diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index c33ce2030c3a8..44f6ba2bcadae 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -229,34 +229,6 @@ Pass arbitrary jvm arguments.
 ./gradlew test -Dtests.jvm.argline="-Djava.security.debug=access,failure"
 ------------------------------
 
-== Backwards Compatibility Tests
-
-Running backwards compatibility tests is disabled by default since it
-requires a release version of elasticsearch to be present on the test system.
-To run backwards compatibility tests untar or unzip a release and run the tests
-with the following command:
-
----------------------------------------------------------------------------
-./gradlew test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch -Dtests.security.manager=false
----------------------------------------------------------------------------
-
-Note that backwards tests must be run with security manager disabled.
-If the elasticsearch release is placed under `./backwards/elasticsearch-x.y.z` the path
-can be omitted:
-
----------------------------------------------------------------------------
-./gradlew test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.security.manager=false
----------------------------------------------------------------------------
-
-To setup the bwc test environment execute the following steps (provided you are
-already in your elasticsearch clone):
-
----------------------------------------------------------------------------
-$ mkdir backwards && cd backwards
-$ curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.2.1.tar.gz
-$ tar -xzf elasticsearch-1.2.1.tar.gz
----------------------------------------------------------------------------
-
 == Running verification tasks
 
 To run all verification tasks, including static checks, unit tests, and integration tests:
@@ -554,25 +526,28 @@ environment variable.
 == Testing backwards compatibility
 
 Backwards compatibility tests exist to test upgrading from each supported version
-to the current version. To run all backcompat tests use:
+to the current version. To run them all use:
 
 -------------------------------------------------
 ./gradlew bwcTest
 -------------------------------------------------
 
-A specific version can be tested as well. For example, to test backcompat with
+A specific version can be tested as well. For example, to test bwc with
 version 5.3.2 run:
 
 -------------------------------------------------
 ./gradlew v5.3.2#bwcTest
 -------------------------------------------------
 
-When running `./gradlew check`, some minimal backcompat checks are run. Which version
-is tested depends on the branch. On master, this will test against the current
-stable branch. On the stable branch, it will test against the latest release
-branch. Finally, on a release branch, it will test against the most recent release.
+Tests are ran for versions that are not yet released but with which the current version will be compatible with.
+These are automatically checked out and built from source.
+See link:./buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java[VersionCollection]
+and link:./distribution/bwc/build.gradle[distribution/bwc/build.gradle] 
+for more information.
+
+When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
-=== BWC Testing against a specific remote/branch
+==== BWC Testing against a specific remote/branch
 
 Sometimes a backward compatibility change spans two versions. A common case is a new functionality
 that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
@@ -597,7 +572,7 @@ will contain your change.
 . Push both branches to your remote repository.
 . Run the tests with `./gradlew check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec.5.x=index_req_bwc_5.x`.
 
-== Skip fetching latest
+==== Skip fetching latest
 
 For some BWC testing scenarios, you want to use the local clone of the
 repository without fetching latest. For these use cases, you can set the system
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
index 591fa400d18da..39a2bdfca0953 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -46,10 +46,6 @@ public final class Allocators {
     private static class NoopGatewayAllocator extends GatewayAllocator {
         public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
 
-        protected NoopGatewayAllocator() {
-            super(Settings.EMPTY);
-        }
-
         @Override
         public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
             // noop
@@ -79,7 +75,7 @@ public static AllocationService createAllocationService(Settings settings) throw
 
     public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
         InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
-        return new AllocationService(settings,
+        return new AllocationService(
             defaultAllocationDeciders(settings, clusterSettings),
             NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
@@ -88,7 +84,7 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl
         IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
         Collection<AllocationDecider> deciders =
             ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList());
-        return new AllocationDeciders(settings, deciders);
+        return new AllocationDeciders(deciders);
 
     }
 
diff --git a/build.gradle b/build.gradle
index 7c7e065671928..3a6196a21b7e4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -39,7 +39,7 @@ if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
 // common maven publishing configuration
 subprojects {
   group = 'org.elasticsearch'
-  version = VersionProperties.elasticsearch.toString()
+  version = VersionProperties.elasticsearch
   description = "Elasticsearch subproject ${project.path}"
 }
 
@@ -103,10 +103,6 @@ subprojects {
  * in a branch if there are only betas and rcs in the branch so we have
  * *something* to test against. */
 VersionCollection versions = new VersionCollection(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
-if (versions.currentVersion != VersionProperties.elasticsearch) {
-  throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
-          "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
-}
 
 // build metadata from previous build, contains eg hashes for bwc builds
 String buildMetadataValue = System.getenv('BUILD_METADATA')
@@ -140,26 +136,16 @@ task verifyVersions {
     if (gradle.startParameter.isOffline()) {
       throw new GradleException("Must run in online mode to verify versions")
     }
-    // Read the list from maven central
-    Node xml
+    // Read the list from maven central. 
+    // Fetch the metadata an parse the xml into Version instances because it's more straight forward here 
+    // rather than bwcVersion ( VersionCollection ).
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
-        xml = new XmlParser().parse(s)
-    }
-    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) })
-
-    // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
-
-    /* Limit the listed versions to those that have been marked as released.
-     * Versions not marked as released don't get the same testing and we want
-     * to make sure that we flip all unreleased versions to released as soon
-     * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot })
-
-    // Finally, compare!
-    if (knownVersions.equals(actualVersions) == false) {
-      throw new GradleException("out-of-date released versions\nActual  :" + actualVersions + "\nExpected:" + knownVersions +
-        "\nUpdate Version.java. Note that Version.CURRENT doesn't count because it is not released.")
+      bwcVersions.compareToAuthoritative(
+              new XmlParser().parse(s)
+                      .versioning.versions.version
+                      .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
+                      .collect { Version.fromString(it) }
+      )
     }
   }
 }
@@ -255,20 +241,17 @@ subprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-
-  bwcVersions.snapshotProjectNames.each { snapshotName ->
-    Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
-    if (snapshot != null ) {
-      String snapshotProject = ":distribution:bwc:${snapshotName}"
-      project(snapshotProject).ext.bwcVersion = snapshot
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject
-      if (snapshot.onOrAfter('6.3.0')) {
-        ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${snapshot}"] = snapshotProject
-        ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${snapshot}"] = snapshotProject
-        ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${snapshot}"] = snapshotProject
-      }
+  // substitute unreleased versions with projects that check out and build locally
+  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
+    Version unreleased = unreleasedVersion.version
+    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${unreleased}"] = snapshotProject
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${unreleased}"] = snapshotProject
+    if (unreleased.onOrAfter('6.3.0')) {
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
     }
   }
 
@@ -303,7 +286,7 @@ subprojects {
     // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
     if (project.plugins.hasPlugin(BuildPlugin) || project.plugins.hasPlugin(PluginBuildPlugin)) {
-      String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { shadowed, dep ->
         if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
@@ -576,11 +559,13 @@ wrapper {
     }
 }
 
-/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
- * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
-    if (project.path.startsWith(':qa')) {
+    /*
+     * Remove assemble/dependenciesInfo on all qa projects because we don't
+     * need to publish artifacts for them.
+     */
+    if (project.name.equals('qa') || project.path.contains(':qa:')) {
       Task assemble = project.tasks.findByName('assemble')
       if (assemble) {
         assemble.enabled = false
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index c201572d4ab5e..ce0162f7bf252 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -44,46 +44,33 @@ if (project == rootProject) {
  *         Propagating version.properties to the rest of the build           *
  *****************************************************************************/
 
-Properties props = new Properties()
-props.load(project.file('version.properties').newDataInputStream())
-version = props.getProperty('elasticsearch')
-boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"));
-if (snapshot) {
-  // we update the version property to reflect if we are building a snapshot or a release build
-  // we write this back out below to load it in the Build.java which will be shown in rest main action
-  // to indicate this being a snapshot build or a release build.
-  version += "-SNAPSHOT"
-  props.put("elasticsearch", version);
-}
-
-File tempPropertiesFile = new File(project.buildDir, "version.properties")
-task writeVersionProperties {
-  inputs.properties(props)
-  outputs.file(tempPropertiesFile)
+// we update the version property to reflect if we are building a snapshot or a release build
+// we write this back out below to load it in the Build.java which will be shown in rest main action
+// to indicate this being a snapshot build or a release build.
+File propsFile = project.file('version.properties')
+Properties props = VersionPropertiesLoader.loadBuildSrcVersion(propsFile)
+version = props.getProperty("elasticsearch")
+processResources {
+  inputs.file(propsFile)
+  // We need to be explicit with the version because we add snapshot and qualifier to it based on properties
+  inputs.property("dynamic_elasticsearch_version", props.getProperty("elasticsearch"))
   doLast {
-    OutputStream stream = Files.newOutputStream(tempPropertiesFile.toPath());
+    Writer writer = file("$destinationDir/version.properties").newWriter()
     try {
-      props.store(stream, "UTF-8");
+      props.store(writer, "Generated version properties")
     } finally {
-      stream.close();
+      writer.close()
     }
   }
 }
 
-processResources {
-  dependsOn writeVersionProperties
-  from tempPropertiesFile
-}
-
-
-if (JavaVersion.current() < JavaVersion.VERSION_1_10) {
-  throw new GradleException('At least Java 10 is required to build elasticsearch gradle tools')
-}
-
 /*****************************************************************************
  *         Java version                                                      *
  *****************************************************************************/
 
+if (JavaVersion.current() < JavaVersion.VERSION_11) {
+  throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
+}
 // Gradle 4.10 does not support setting this to 11 yet
 targetCompatibility = "10"
 sourceCompatibility = "10"
@@ -238,3 +225,42 @@ if (project != rootProject) {
     generatePomFileForPluginMavenPublication.enabled = false
   }
 }
+
+// Define this here because we need it early.
+class VersionPropertiesLoader {
+  static Properties loadBuildSrcVersion(File input) throws IOException {
+    Properties props = new Properties();
+    InputStream is = new FileInputStream(input)
+    try {
+      props.load(is)
+    } finally {
+      is.close()
+    }
+    loadBuildSrcVersion(props, System.getProperties())
+    return props
+  }
+
+  protected static void loadBuildSrcVersion(Properties loadedProps, Properties systemProperties) {
+    String elasticsearch = loadedProps.getProperty("elasticsearch")
+    if (elasticsearch == null) {
+      throw new IllegalStateException("Elasticsearch version is missing from properties.")
+    }
+    if (elasticsearch.matches("[0-9]+\\.[0-9]+\\.[0-9]+") == false) {
+      throw new IllegalStateException(
+              "Expected elasticsearch version to be numbers only of the form  X.Y.Z but it was: " +
+                      elasticsearch
+      )
+    }
+    String qualifier = systemProperties.getProperty("build.version_qualifier", "");
+    if (qualifier.isEmpty() == false) {
+      if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
+        throw new IllegalStateException("Invalid qualifier: " + qualifier)
+      }
+      elasticsearch += "-" + qualifier
+    }
+    if ("true".equals(systemProperties.getProperty("build.snapshot", "true"))) {
+      elasticsearch += "-SNAPSHOT"
+    }
+    loadedProps.put("elasticsearch", elasticsearch)
+  }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index f42dde73f73de..d9af2f25dcba0 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -242,11 +242,12 @@ class BuildPlugin implements Plugin<Project> {
             compilerJavaHome = findJavaHome(compilerJavaProperty)
         }
         if (compilerJavaHome == null) {
-            if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
+            if (System.getProperty("idea.executable") != null || System.getProperty("eclipse.launcher") != null) {
                 // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
                 return Jvm.current().javaHome
             } else {
                 throw new GradleException(
+                        " " + System.getProperties().toString() + " " +
                         "JAVA_HOME must be set to build Elasticsearch. " +
                                 "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
                                 "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
@@ -696,18 +697,12 @@ class BuildPlugin implements Plugin<Project> {
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
             // fixup the jar manifest
             jarTask.doFirst {
-                final Version versionWithoutSnapshot = new Version(
-                        VersionProperties.elasticsearch.major,
-                        VersionProperties.elasticsearch.minor,
-                        VersionProperties.elasticsearch.revision,
-                        VersionProperties.elasticsearch.suffix,
-                        false)
                 // this doFirst is added before the info plugin, therefore it will run
                 // after the doFirst added by the info plugin, and we can override attributes
                 jarTask.manifest.attributes(
-                        'X-Compile-Elasticsearch-Version': versionWithoutSnapshot,
+                        'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch.replace("-SNAPSHOT", ""),
                         'X-Compile-Lucene-Version': VersionProperties.lucene,
-                        'X-Compile-Elasticsearch-Snapshot': VersionProperties.elasticsearch.isSnapshot(),
+                        'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(),
                         'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
                         'Build-Java-Version': project.compilerJavaVersion)
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy
deleted file mode 100644
index c69958ddad4b3..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle
-
-import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
-
-import java.util.regex.Matcher
-
-/**
- * The collection of version constants declared in Version.java, for use in BWC testing.
- *
- *  if major+1 released: released artifacts from $version down to major-1.highestMinor.highestPatch, none of these should be snapshots, period.
- *  if major+1 unreleased:
- *  - if released:
- *   -- caveat 0: snapshot for the major-1.highestMinor.highestPatch
- *  - if unreleased:
- *   -- caveat 0: snapshot for the major-1.highestMinor.highestPatch
- *   -- caveat 1: every same major lower minor branch should also be tested if its released, and if not, its a snapshot. There should only be max 2 of these.
- *   -- caveat 2: the largest released minor branch before the unreleased minor should also be a snapshot
- *   -- caveat 3: if the current version is a different major than the previous rules apply to major - 1 of the current version
- *
- * Please note that the caveat's also correspond with the 4 types of snapshots.
- * - Caveat 0 - always maintenanceBugfixSnapshot.
- * - Caveat 1 - This is tricky. If caveat 3 applies, the highest matching value is nextMinorSnapshot, if there is another it is the stagedMinorSnapshot.
- *              If caveat 3 does not apply then the only possible value is the stagedMinorSnapshot.
- * - Caveat 2 - always nextBugfixSnapshot
- * - Caveat 3 - this only changes the applicability of Caveat 1
- *
- * Notes on terminology:
- * - The case for major+1 being released is accomplished through the isReleasableBranch value. If this is false, then the branch is no longer
- *   releasable, meaning not to test against any snapshots.
- * - Released is defined as having > 1 suffix-free version in a major.minor series. For instance, only 6.2.0 means unreleased, but a
- *   6.2.0 and 6.2.1 mean that 6.2.0 was released already.
- */
-class VersionCollection {
-
-    private final List<Version> versions
-    Version nextMinorSnapshot
-    Version stagedMinorSnapshot
-    Version nextBugfixSnapshot
-    Version maintenanceBugfixSnapshot
-    final Version currentVersion
-    private final TreeSet<Version> versionSet = new TreeSet<>()
-    final List<String> snapshotProjectNames = ['next-minor-snapshot',
-                                               'staged-minor-snapshot',
-                                               'next-bugfix-snapshot',
-                                               'maintenance-bugfix-snapshot']
-
-    // When we roll 8.0 its very likely these will need to be extracted from this class
-    private final boolean isReleasableBranch = true
-
-    /**
-     * Construct a VersionCollection from the lines of the Version.java file. The basic logic for the following is pretty straight forward.
-
-     * @param versionLines The lines of the Version.java file.
-     */
-    VersionCollection(List<String> versionLines) {
-        final boolean buildSnapshot = System.getProperty("build.snapshot", "true") == "true"
-
-        List<Version> versions = []
-        // This class should be converted wholesale to use the treeset
-
-        for (final String line : versionLines) {
-            final Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_alpha\d+|_beta\d+|_rc\d+)? .*/
-            if (match.matches()) {
-                final Version foundVersion = new Version(
-                        Integer.parseInt(match.group(1)), Integer.parseInt(match.group(2)),
-                        Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false)
-                safeAddToSet(foundVersion)
-            }
-        }
-
-        if (versionSet.empty) {
-            throw new GradleException("Unexpectedly found no version constants in Versions.java")
-        }
-
-        // If the major version has been released, then remove all of the alpha/beta/rc versions that exist in the set
-        versionSet.removeAll { it.suffix.isEmpty() == false && isMajorReleased(it, versionSet) }
-
-        // set currentVersion
-        Version lastVersion = versionSet.last()
-        currentVersion = new Version(lastVersion.major, lastVersion.minor, lastVersion.revision, lastVersion.suffix, buildSnapshot)
-
-        // remove all of the potential alpha/beta/rc from the currentVersion
-        versionSet.removeAll {
-            it.suffix.isEmpty() == false &&
-            it.major == currentVersion.major &&
-            it.minor == currentVersion.minor &&
-            it.revision == currentVersion.revision }
-
-        // re-add the currentVersion to the set
-        versionSet.add(currentVersion)
-
-        if (isReleasableBranch) {
-            if (isReleased(currentVersion)) {
-                // caveat 0 - if the minor has been released then it only has a maintenance version
-                // go back 1 version to get the last supported snapshot version of the line, which is a maint bugfix
-                Version highestMinor = getHighestPreviousMinor(currentVersion.major)
-                maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
-            } else {
-                // caveat 3 - if our currentVersion is a X.0.0, we need to check X-1 minors to see if they are released
-                if (currentVersion.minor == 0) {
-                    for (Version version: getMinorTips(currentVersion.major - 1)) {
-                        if (isReleased(version) == false) {
-                            // caveat 1 - This should only ever contain 2 non released branches in flight. An example is 6.x is frozen,
-                            // and 6.2 is cut but not yet released there is some simple logic to make sure that in the case of more than 2,
-                            // it will bail. The order is that the minor snapshot is fulfilled first, and then the staged minor snapshot
-                            if (nextMinorSnapshot == null) {
-                                // it has not been set yet
-                                nextMinorSnapshot = replaceAsSnapshot(version)
-                            } else if (stagedMinorSnapshot == null) {
-                                stagedMinorSnapshot = replaceAsSnapshot(version)
-                            } else {
-                                throw new GradleException("More than 2 snapshot version existed for the next minor and staged (frozen) minors.")
-                            }
-                        } else {
-                            // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break
-                            nextBugfixSnapshot = replaceAsSnapshot(version)
-                            // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1
-                            break
-                        }
-                    }
-                    // caveat 0 - now dip back 2 versions to get the last supported snapshot version of the line
-                    Version highestMinor = getHighestPreviousMinor(currentVersion.major - 1)
-                    maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
-                } else {
-                    // caveat 3 did not apply. version is not a X.0.0, so we are somewhere on a X.Y line
-                    // only check till minor == 0 of the major
-                    for (Version version: getMinorTips(currentVersion.major)) {
-                        if (isReleased(version) == false) {
-                            // caveat 1 - This should only ever contain 0 or 1 branch in flight. An example is 6.x is frozen, and 6.2 is cut
-                            // but not yet released there is some simple logic to make sure that in the case of more than 1, it will bail
-                            if (stagedMinorSnapshot == null) {
-                                stagedMinorSnapshot = replaceAsSnapshot(version)
-                            } else {
-                                throw new GradleException("More than 1 snapshot version existed for the staged (frozen) minors.")
-                            }
-                        } else {
-                            // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break
-                            nextBugfixSnapshot = replaceAsSnapshot(version)
-                            // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1
-                            break
-                        }
-                    }
-                    // caveat 0 - now dip back 1 version to get the last supported snapshot version of the line
-                    Version highestMinor = getHighestPreviousMinor(currentVersion.major)
-                    maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
-                }
-            }
-        }
-
-        this.versions = Collections.unmodifiableList(versionSet.toList())
-    }
-
-    /**
-     * @return The list of versions read from the Version.java file
-     */
-    List<Version> getVersions() {
-        return versions
-    }
-
-    /**
-     * Index compat supports 1 previous entire major version. For instance, any 6.x test for this would test all of 5 up to that 6.x version
-     *
-     * @return All earlier versions that should be tested for index BWC with the current version.
-     */
-    List<Version> getIndexCompatible() {
-        int actualMajor = (currentVersion.major == 5 ? 2 : currentVersion.major - 1)
-        return versionSet
-            .tailSet(Version.fromString("${actualMajor}.0.0"))
-            .headSet(currentVersion)
-            .asList()
-    }
-
-    /**
-     * Ensures the types of snapshot are not null and are also in the index compat list
-     */
-    List<Version> getSnapshotsIndexCompatible() {
-        List<Version> compatSnapshots = []
-        List<Version> allCompatVersions = getIndexCompatible()
-        if (allCompatVersions.contains(nextMinorSnapshot)) {
-            compatSnapshots.add(nextMinorSnapshot)
-        }
-        if (allCompatVersions.contains(stagedMinorSnapshot)) {
-            compatSnapshots.add(stagedMinorSnapshot)
-        }
-        if (allCompatVersions.contains(nextBugfixSnapshot)) {
-            compatSnapshots.add(nextBugfixSnapshot)
-        }
-        if (allCompatVersions.contains(maintenanceBugfixSnapshot)) {
-            compatSnapshots.add(maintenanceBugfixSnapshot)
-        }
-
-        return compatSnapshots
-    }
-
-    /**
-     * Wire compat supports the last minor of the previous major. For instance, any 6.x test would test 5.6 up to that 6.x version
-     *
-     * @return All earlier versions that should be tested for wire BWC with the current version.
-     */
-    List<Version> getWireCompatible() {
-        // Get the last minor of the previous major
-        Version lowerBound = getHighestPreviousMinor(currentVersion.major)
-        return versionSet
-            .tailSet(Version.fromString("${lowerBound.major}.${lowerBound.minor}.0"))
-            .headSet(currentVersion)
-            .toList()
-    }
-
-    /**
-     * Ensures the types of snapshot are not null and are also in the wire compat list
-     */
-    List<Version> getSnapshotsWireCompatible() {
-        List<Version> compatSnapshots = []
-        List<Version> allCompatVersions = getWireCompatible()
-        if (allCompatVersions.contains(nextMinorSnapshot)) {
-            compatSnapshots.add(nextMinorSnapshot)
-        }
-        if (allCompatVersions.contains(stagedMinorSnapshot)) {
-            compatSnapshots.add(stagedMinorSnapshot)
-        }
-        if (allCompatVersions.contains(nextBugfixSnapshot)) {
-            compatSnapshots.add(nextBugfixSnapshot)
-        }
-        if (allCompatVersions.contains(maintenanceBugfixSnapshot)) {
-            compatSnapshots.add(maintenanceBugfixSnapshot)
-        }
-        // There was no wire compat for the 2.x line
-        compatSnapshots.removeAll {it.major == 2}
-
-        return compatSnapshots
-    }
-
-    /**
-     * Grabs the proper snapshot based on the name passed in. These names should correspond with gradle project names under bwc. If you
-     * are editing this if/else it is only because you added another project under :distribution:bwc. Do not modify this method or its
-     * reasoning for throwing the exception unless you are sure that it will not harm :distribution:bwc.
-     */
-    Version getSnapshotForProject(String snapshotProjectName) {
-        if (snapshotProjectName == 'next-minor-snapshot') {
-            return nextMinorSnapshot
-        } else if (snapshotProjectName == 'staged-minor-snapshot') {
-            return stagedMinorSnapshot
-        } else if (snapshotProjectName == 'maintenance-bugfix-snapshot') {
-            return maintenanceBugfixSnapshot
-        } else if (snapshotProjectName == 'next-bugfix-snapshot') {
-            return nextBugfixSnapshot
-        } else {
-            throw new InvalidUserDataException("Unsupported project name ${snapshotProjectName}")
-        }
-    }
-
-    /**
-     * Uses basic logic about our releases to determine if this version has been previously released
-     */
-    private boolean isReleased(Version version) {
-        return version.revision > 0
-    }
-
-    /**
-     * Validates that the count of non suffixed (alpha/beta/rc) versions in a given major to major+1 is greater than 1.
-     * This means that there is more than just a major.0.0 or major.0.0-alpha in a branch to signify it has been prevously released.
-     */
-    private boolean isMajorReleased(Version version, TreeSet<Version> items) {
-        return items
-            .tailSet(Version.fromString("${version.major}.0.0"))
-            .headSet(Version.fromString("${version.major + 1}.0.0"))
-            .count { it.suffix.isEmpty() }  // count only non suffix'd versions as actual versions that may be released
-            .intValue() > 1
-    }
-
-    /**
-     * Gets the largest version previous major version based on the nextMajorVersion passed in.
-     * If you have a list [5.0.2, 5.1.2, 6.0.1, 6.1.1] and pass in 6 for the nextMajorVersion, it will return you 5.1.2
-     */
-    private Version getHighestPreviousMinor(Integer nextMajorVersion) {
-        return versionSet.headSet(Version.fromString("${nextMajorVersion}.0.0")).last()
-    }
-
-    /**
-     * Helper function for turning a version into a snapshot version, removing and readding it to the tree
-     */
-    private Version replaceAsSnapshot(Version version) {
-        versionSet.remove(version)
-        Version snapshotVersion = new Version(version.major, version.minor, version.revision, version.suffix, true)
-        safeAddToSet(snapshotVersion)
-        return snapshotVersion
-    }
-
-    /**
-     * Safely adds a value to the treeset, or bails if the value already exists.
-     * @param version
-     */
-    private void safeAddToSet(Version version) {
-        if (versionSet.add(version) == false) {
-            throw new GradleException("Versions.java contains duplicate entries for ${version}")
-        }
-    }
-
-    /**
-     * Gets the entire set of major.minor.* given those parameters.
-     */
-    private SortedSet<Version> getMinorSetForMajor(Integer major, Integer minor) {
-        return versionSet
-            .tailSet(Version.fromString("${major}.${minor}.0"))
-            .headSet(Version.fromString("${major}.${minor + 1}.0"))
-    }
-
-    /**
-     * Gets the entire set of major.* to the currentVersion
-     */
-    private SortedSet<Version> getMajorSet(Integer major) {
-        return versionSet
-            .tailSet(Version.fromString("${major}.0.0"))
-            .headSet(currentVersion)
-    }
-
-    /**
-     * Gets the tip of each minor set and puts it in a list.
-     *
-     * examples:
-     *  [1.0.0, 1.1.0, 1.1.1, 1.2.0, 1.3.1] will return [1.0.0, 1.1.1, 1.2.0, 1.3.1]
-     *  [1.0.0, 1.0.1, 1.0.2, 1.0.3, 1.0.4] will return [1.0.4]
-     */
-    private List<Version> getMinorTips(Integer major) {
-        TreeSet<Version> majorSet = getMajorSet(major)
-        List<Version> minorList = new ArrayList<>()
-        for (int minor = majorSet.last().minor; minor >= 0; minor--) {
-            TreeSet<Version> minorSetInMajor = getMinorSetForMajor(major, minor)
-            minorList.add(minorSetInMajor.last())
-        }
-        return minorList
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
index 9b2b1ca215673..881fce443a792 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -42,7 +42,7 @@ public class DocsTestPlugin extends RestTestPlugin {
              * to the version being built for testing but needs to resolve to
              * the last released version for docs. */
             '\\{version\\}':
-                VersionProperties.elasticsearch.toString().replace('-SNAPSHOT', ''),
+                VersionProperties.elasticsearch.replace('-SNAPSHOT', ''),
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
             '\\{build_flavor\\}' :
                 project.integTestCluster.distribution.startsWith('oss-') ? 'oss' : 'default',
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 9828f68991f9b..440127c456247 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -96,7 +96,7 @@ public class PluginBuildPlugin extends BuildPlugin {
                     project.pluginProperties.extension.name + "-client"
             )
             project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
-                generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.version}.pom"
+                generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.versions.elasticsearch}.pom"
             }
         } else {
             project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
index 9588f77a71db7..633647514ed7d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy
@@ -76,7 +76,7 @@ class PluginPropertiesTask extends Copy {
             'name': extension.name,
             'description': extension.description,
             'version': stringSnap(extension.version),
-            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch.toString()),
+            'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
             'extendedPlugins': extension.extendedPlugins.join(','),
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 0e706aa5956f1..b5476ea96621b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -22,6 +22,7 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
@@ -220,7 +221,7 @@ class PrecommitTasks {
     private static Task configureLoggerUsage(Project project) {
         project.configurations.create('loggerUsagePlugin')
         project.dependencies.add('loggerUsagePlugin',
-                "org.elasticsearch.test:logger-usage:${org.elasticsearch.gradle.VersionProperties.elasticsearch}")
+                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}")
         return project.tasks.create('loggerUsageCheck', LoggerUsageTask.class) {
             classpath = project.configurations.loggerUsagePlugin
             javaHome = project.runtimeJavaHome
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 4f3ade2dae296..52ed30880d374 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -98,7 +98,7 @@ class ClusterFormationTasks {
             // from here on everything else works the same as if it's the current version, we fetch the BWC version
             // from mirrors using gradles built-in mechanism etc.
 
-            configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
+            configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion.toString())
             for (Map.Entry<String, Object> entry : config.plugins.entrySet()) {
                 configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion)
             }
@@ -109,9 +109,12 @@ class ClusterFormationTasks {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
             // for each of those nodes we might have a different configuration
             final Configuration distro
-            final Version elasticsearchVersion
+            final String elasticsearchVersion
             if (i < config.numBwcNodes) {
-                elasticsearchVersion = config.bwcVersion
+                elasticsearchVersion = config.bwcVersion.toString()
+                if (project.bwcVersions.unreleased.contains(config.bwcVersion)) {
+                    elasticsearchVersion += "-SNAPSHOT"
+                }
                 distro = bwcDistro
             } else {
                 elasticsearchVersion = VersionProperties.elasticsearch
@@ -130,8 +133,10 @@ class ClusterFormationTasks {
     }
 
     /** Adds a dependency on the given distribution */
-    static void configureDistributionDependency(Project project, String distro, Configuration configuration, Version elasticsearchVersion) {
-        if (elasticsearchVersion.before('6.3.0') && distro.startsWith('oss-')) {
+    static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        if (Version.fromString(elasticsearchVersion).before('6.3.0') &&
+                distro.startsWith('oss-')
+        ) {
             distro = distro.substring('oss-'.length())
         }
         String packaging = distro
@@ -201,7 +206,7 @@ class ClusterFormationTasks {
         setup = configureAddKeystoreFileTasks(prefix, project, setup, node)
 
         if (node.config.plugins.isEmpty() == false) {
-            if (node.nodeVersion == VersionProperties.elasticsearch) {
+            if (node.nodeVersion == Version.fromString(VersionProperties.elasticsearch)) {
                 setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node, prefix)
             } else {
                 setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node, prefix)
@@ -558,7 +563,7 @@ class ClusterFormationTasks {
 
     static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
         final FileCollection pluginZip;
-        if (node.nodeVersion != VersionProperties.elasticsearch) {
+        if (node.nodeVersion != Version.fromString(VersionProperties.elasticsearch)) {
             pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
         } else {
             pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName))
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index a0100767d8552..4c8546bcc8928 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -112,7 +112,7 @@ class NodeInfo {
     Version nodeVersion
 
     /** Holds node configuration for part of a test cluster. */
-    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, Version nodeVersion, File sharedDir) {
+    NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
         this.config = config
         this.nodeNum = nodeNum
         this.project = project
@@ -124,7 +124,7 @@ class NodeInfo {
         }
         baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
-        this.nodeVersion = nodeVersion
+        this.nodeVersion = Version.fromString(nodeVersion)
         homeDir = homeDir(baseDir, config.distribution, nodeVersion)
         pathConf = pathConf(baseDir, config.distribution, nodeVersion)
         if (config.dataDir != null) {
@@ -173,11 +173,11 @@ class NodeInfo {
         }
 
 
-        if (nodeVersion.before("6.2.0")) {
+        if (this.nodeVersion.before("6.2.0")) {
             javaVersion = 8
-        } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) {
+        } else if (this.nodeVersion.onOrAfter("6.2.0") && this.nodeVersion.before("6.3.0")) {
             javaVersion = 9
-        } else if (nodeVersion.onOrAfter("6.3.0") && nodeVersion.before("6.5.0")) {
+        } else if (this.nodeVersion.onOrAfter("6.3.0") && this.nodeVersion.before("6.5.0")) {
             javaVersion = 10
         }
 
@@ -200,7 +200,7 @@ class NodeInfo {
         else {
             env.put('ES_PATH_CONF', pathConf)
         }
-        if (nodeVersion.major == 5) {
+        if (this.nodeVersion.major == 5) {
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 /*
                  * We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
@@ -312,7 +312,7 @@ class NodeInfo {
     }
 
     /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro, Version nodeVersion) {
+    static File homeDir(File baseDir, String distro, String nodeVersion) {
         String path
         switch (distro) {
             case 'integ-test-zip':
@@ -332,7 +332,7 @@ class NodeInfo {
         return new File(baseDir, path)
     }
 
-    static File pathConf(File baseDir, String distro, Version nodeVersion) {
+    static File pathConf(File baseDir, String distro, String nodeVersion) {
         switch (distro) {
             case 'integ-test-zip':
             case 'zip':
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java
index f9f1791a9569f..460df76ace198 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java
@@ -12,44 +12,20 @@ public final class Version implements Comparable<Version> {
     private final int minor;
     private final int revision;
     private final int id;
-    private final boolean snapshot;
-    /**
-     * Suffix on the version name.
-     */
-    private final String suffix;
 
     private static final Pattern pattern =
             Pattern.compile("(\\d)+\\.(\\d+)\\.(\\d+)(-alpha\\d+|-beta\\d+|-rc\\d+)?(-SNAPSHOT)?");
 
-    public Version(int major, int minor, int revision, String suffix, boolean snapshot) {
+    public Version(int major, int minor, int revision) {
         Objects.requireNonNull(major, "major version can't be null");
         Objects.requireNonNull(minor, "minor version can't be null");
         Objects.requireNonNull(revision, "revision version can't be null");
         this.major = major;
         this.minor = minor;
         this.revision = revision;
-        this.snapshot = snapshot;
-        this.suffix = suffix == null ? "" : suffix;
-
-        int suffixOffset = 0;
-        if (this.suffix.isEmpty()) {
-            // no suffix will be considered smaller, uncomment to change that
-            // suffixOffset = 100;
-        } else {
-            if (this.suffix.contains("alpha")) {
-                suffixOffset += parseSuffixNumber(this.suffix.substring(6));
-            } else if (this.suffix.contains("beta")) {
-                suffixOffset += 25 + parseSuffixNumber(this.suffix.substring(5));
-            } else if (this.suffix.contains("rc")) {
-                suffixOffset += 50 + parseSuffixNumber(this.suffix.substring(3));
-            }
-            else {
-                throw new IllegalArgumentException("Suffix must contain one of: alpha, beta or rc");
-            }
-        }
 
         // currently snapshot is not taken into account
-        this.id = major * 10000000 + minor * 100000 + revision * 1000 + suffixOffset * 10 /*+ (snapshot ? 1 : 0)*/;
+        this.id = major * 10000000 + minor * 100000 + revision * 1000;
     }
 
     private static int parseSuffixNumber(String substring) {
@@ -71,17 +47,13 @@ public static Version fromString(final String s) {
         return new Version(
                 Integer.parseInt(matcher.group(1)),
                 parseSuffixNumber(matcher.group(2)),
-                parseSuffixNumber(matcher.group(3)),
-                matcher.group(4),
-                matcher.group(5) != null
+                parseSuffixNumber(matcher.group(3))
         );
     }
 
     @Override
     public String toString() {
-        final String snapshotStr = snapshot ? "-SNAPSHOT" : "";
-        return String.valueOf(getMajor()) + "." + String.valueOf(getMinor()) + "." + String.valueOf(getRevision()) +
-                (suffix == null ? "" : suffix) + snapshotStr;
+        return String.valueOf(getMajor()) + "." + String.valueOf(getMinor()) + "." + String.valueOf(getRevision());
     }
 
     public boolean before(Version compareTo) {
@@ -116,19 +88,6 @@ public boolean after(String compareTo) {
         return after(fromString(compareTo));
     }
 
-    public boolean onOrBeforeIncludingSuffix(Version otherVersion) {
-        if (id != otherVersion.getId()) {
-            return id < otherVersion.getId();
-        }
-
-        if (suffix.equals("")) {
-            return otherVersion.getSuffix().equals("");
-        }
-
-
-        return otherVersion.getSuffix().equals("") || suffix.compareTo(otherVersion.getSuffix()) < 0;
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -136,16 +95,12 @@ public boolean equals(Object o) {
         Version version = (Version) o;
         return major == version.major &&
                 minor == version.minor &&
-                revision == version.revision &&
-                id == version.id &&
-                snapshot == version.snapshot &&
-                Objects.equals(suffix, version.suffix);
+                revision == version.revision;
     }
 
     @Override
     public int hashCode() {
-
-        return Objects.hash(major, minor, revision, id, snapshot, suffix);
+        return Objects.hash(major, minor, revision, id);
     }
 
     public int getMajor() {
@@ -164,14 +119,6 @@ protected int getId() {
         return id;
     }
 
-    public boolean isSnapshot() {
-        return snapshot;
-    }
-
-    public String getSuffix() {
-        return suffix;
-    }
-
     @Override
     public int compareTo(Version other) {
         return Integer.compare(getId(), other.getId());
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java
new file mode 100644
index 0000000000000..1cf2fd9e1037c
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+
+/**
+ * A container for elasticsearch supported version information used in BWC testing.
+ *
+ * Parse the Java source file containing the versions declarations and use the known rules to figure out which are all
+ * the version the current one is wire and index compatible with.
+ * On top of this, figure out which of these are unreleased and provide the branch they can be built from.
+ *
+ * Note that in this context, currentVersion is the unreleased version this build operates on.
+ * At any point in time there will surely be four such unreleased versions being worked on,
+ * thus currentVersion will be one of these.
+ *
+ * Considering:
+ * <dl>
+ *     <dt>M, M &gt; 0</dt>
+ *     <dd>last released major</dd>
+ *     <dt>N, N &gt; 0</dt>
+ *     <dd>last released minor</dd>
+ * </dl>
+ *
+ * <ul>
+ * <li>the unreleased <b>major</b>, M+1.0.0 on the `master` branch</li>
+ * <li>the unreleased <b>minor</b>,  M.N.0 on the `M.x` (x is literal) branch</li>
+ * <li>the unreleased <b>bugfix</b>, M.N.c (c &gt; 0) on the `M.b` branch</li>
+ * <li>the unreleased <b>maintenance</b>, M-1.d.e ( d &gt; 0, e &gt; 0) on the `(M-1).d` branch</li>
+ * </ul>
+ * In addition to these, there will be a fifth one when a minor reaches feature freeze, we call this the <i>staged</i>
+ * version:
+ * <ul>
+ * <li>the unreleased <b>staged</b>, M.N-2.0 (N &gt; 2) on the `M.(N-2)` branch</li>
+ * </ul>
+ *
+ * Each build is only concerned with versions before it, as those are the ones that need to be tested
+ * for backwards compatibility. We never look forward, and don't add forward facing version number to branches of previous
+ * version.
+ *
+ * Each branch has a current version, and expected compatible versions are parsed from the server code's Version` class.
+ * We can reliably figure out which the unreleased versions are due to the convention of always adding the next unreleased
+ * version number to server in all branches when a version is released.
+ * E.x when M.N.c is released M.N.c+1 is added to the Version class mentioned above in all the following branches:
+ *  `M.b`, `M.x` and `master` so we can reliably assume that the leafs of the version tree are unreleased.
+ * This convention is enforced by checking the versions we consider to be unreleased against an
+ * authoritative source (maven central).
+ * We are then able to map the unreleased version to branches in git and Gradle projects that are capable of checking
+ * out and building them, so we can include these in the testing plan as well.
+ */
+public class VersionCollection {
+
+    private static final Pattern LINE_PATTERN = Pattern.compile(
+        "\\W+public static final Version V_(\\d+)_(\\d+)_(\\d+)(_alpha\\d+|_beta\\d+|_rc\\d+)? .*"
+    );
+
+    private final Version currentVersion;
+    private final Map<Integer, List<Version>> groupByMajor;
+
+    public class UnreleasedVersionInfo {
+        public final Version version;
+        public final String branch;
+        public final String gradleProjectName;
+
+        UnreleasedVersionInfo(Version version, String branch, String gradleProjectName) {
+            this.version = version;
+            this.branch = branch;
+            this.gradleProjectName = gradleProjectName;
+        }
+    }
+
+    public VersionCollection(List<String> versionLines) {
+        this(versionLines, Version.fromString(VersionProperties.getElasticsearch()));
+    }
+
+    protected VersionCollection(List<String> versionLines, Version currentVersionProperty) {
+        groupByMajor = versionLines.stream()
+            .map(LINE_PATTERN::matcher)
+            .filter(Matcher::matches)
+            .map(match -> new Version(
+                Integer.parseInt(match.group(1)),
+                Integer.parseInt(match.group(2)),
+                Integer.parseInt(match.group(3))
+            ))
+            .sorted()
+            .distinct()
+            .collect(Collectors.groupingBy(Version::getMajor, Collectors.toList()));
+
+        if (groupByMajor.isEmpty()) {
+            throw new IllegalArgumentException("Could not parse any versions");
+        }
+
+        currentVersion = getLatestVersionByKey(
+            groupByMajor,
+            groupByMajor.keySet().stream().max(Integer::compareTo)
+                .orElseThrow(() -> new IllegalStateException("Unexpected number of versions in collection"))
+        );
+
+        assertCurrentVersionMatchesParsed(currentVersionProperty);
+
+        assertNoOlderThanTwoMajors();
+    }
+
+    private void assertNoOlderThanTwoMajors() {
+        Set<Integer> majors = groupByMajor.keySet();
+        if (majors.size() != 2 && currentVersion.getMinor() != 0 && currentVersion.getRevision() != 0) {
+            throw new IllegalStateException(
+                "Expected exactly 2 majors in parsed versions but found: " + majors
+            );
+        }
+    }
+
+    private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
+        if (currentVersionProperty.equals(currentVersion) == false) {
+            throw new IllegalStateException(
+                "Parsed versions latest version does not match the one configured in build properties. " +
+                    "Parsed latest version is " + currentVersion + " but the build has " +
+                    currentVersionProperty
+            );
+        }
+    }
+
+    public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
+        getUnreleased().stream()
+            .filter(version -> version.equals(currentVersion) == false)
+            .forEach(version -> consumer.accept(
+                new UnreleasedVersionInfo(
+                    version,
+                    getBranchFor(version),
+                    getGradleProjectNameFor(version)
+                )
+            ));
+    }
+
+    private String getGradleProjectNameFor(Version version) {
+        if (version.equals(currentVersion)) {
+            throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build.");
+        }
+        Map<Integer, List<Version>> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor();
+
+        if (version.getRevision() == 0) {
+            if (releasedMajorGroupedByMinor
+                .get(releasedMajorGroupedByMinor.keySet().stream().max(Integer::compareTo).orElse(0))
+                .contains(version)) {
+                return "minor";
+            } else {
+                return "staged";
+            }
+        } else {
+            if (releasedMajorGroupedByMinor
+                .getOrDefault(version.getMinor(), emptyList())
+                .contains(version)) {
+                return "bugfix";
+            } else {
+                return "maintenance";
+            }
+        }
+    }
+
+    private String getBranchFor(Version version) {
+        switch (getGradleProjectNameFor(version)) {
+            case "minor":
+                return version.getMajor() + ".x";
+            case "staged":
+            case "maintenance":
+            case "bugfix":
+                return version.getMajor() + "." + version.getMinor();
+            default:
+                throw new IllegalStateException("Unexpected Gradle project name");
+        }
+    }
+
+    public List<Version> getUnreleased() {
+        List<Version> unreleased = new ArrayList<>();
+        // The current version is being worked, is always unreleased
+        unreleased.add(currentVersion);
+
+        // the tip of the previous major is unreleased for sure, be it a minor or a bugfix
+        unreleased.add(getLatestVersionByKey(this.groupByMajor, currentVersion.getMajor() - 1));
+
+        final Map<Integer, List<Version>> groupByMinor = getReleasedMajorGroupedByMinor();
+        int greatestMinor = groupByMinor.keySet().stream().max(Integer::compareTo).orElse(0);
+
+        // the last bugfix for this minor series is always unreleased
+        unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor));
+
+        if (groupByMinor.get(greatestMinor).size() == 1) {
+            // we found an unreleased minor
+            unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 1));
+            if (groupByMinor.getOrDefault(greatestMinor - 1, emptyList()).size() == 1) {
+                // we found that the previous minor is staged but not yet released
+                // in this case, the minor before that has a bugfix
+                unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+            }
+        }
+
+        return unmodifiableList(
+            unreleased.stream()
+                .sorted()
+                .distinct()
+                .collect(Collectors.toList())
+        );
+    }
+
+    private Version getLatestVersionByKey(Map<Integer, List<Version>> groupByMajor, int key) {
+        return groupByMajor.getOrDefault(key, emptyList()).stream()
+            .max(Version::compareTo)
+            .orElseThrow(() -> new IllegalStateException("Unexpected number of versions in collection"));
+    }
+
+    private Map<Integer, List<Version>> getReleasedMajorGroupedByMinor() {
+        List<Version> currentMajorVersions = groupByMajor.get(currentVersion.getMajor());
+        List<Version> previousMajorVersions = groupByMajor.get(currentVersion.getMajor() - 1);
+
+        final Map<Integer, List<Version>> groupByMinor;
+        if (currentMajorVersions.size() == 1) {
+            // Current is an unreleased major: x.0.0 so we have to look for other unreleased versions in the previous major
+            groupByMinor = previousMajorVersions.stream()
+                .collect(Collectors.groupingBy(Version::getMinor, Collectors.toList()));
+        } else {
+            groupByMinor = currentMajorVersions.stream()
+                .collect(Collectors.groupingBy(Version::getMinor, Collectors.toList()));
+        }
+        return groupByMinor;
+    }
+
+    public void compareToAuthoritative(List<Version> authoritativeReleasedVersions) {
+        Set<Version> notReallyReleased = new HashSet<>(getReleased());
+        notReallyReleased.removeAll(authoritativeReleasedVersions);
+        if (notReallyReleased.isEmpty() == false) {
+            throw new IllegalStateException(
+                "out-of-date released versions" +
+                    "\nFollowing versions are not really released, but the build thinks they are: " + notReallyReleased
+            );
+        }
+
+        Set<Version> incorrectlyConsideredUnreleased = new HashSet<>(authoritativeReleasedVersions);
+        incorrectlyConsideredUnreleased.retainAll(getUnreleased());
+        if (incorrectlyConsideredUnreleased.isEmpty() == false) {
+            throw new IllegalStateException(
+                "out-of-date released versions" +
+                    "\nBuild considers versions unreleased, " +
+                    "but they are released according to an authoritative source: " + incorrectlyConsideredUnreleased +
+                    "\nThe next versions probably needs to be added to Version.java (CURRENT doesn't count)."
+            );
+        }
+    }
+
+    private List<Version> getReleased() {
+        List<Version> unreleased = getUnreleased();
+        return groupByMajor.values().stream()
+            .flatMap(Collection::stream)
+            .filter(each -> unreleased.contains(each) == false)
+            .collect(Collectors.toList());
+    }
+
+    public List<Version> getIndexCompatible() {
+        return unmodifiableList(
+            Stream.concat(
+                groupByMajor.get(currentVersion.getMajor() - 1).stream(),
+                groupByMajor.get(currentVersion.getMajor()).stream()
+            )
+                .filter(version -> version.equals(currentVersion) == false)
+                .collect(Collectors.toList())
+        );
+    }
+
+    public List<Version> getWireCompatible() {
+        List<Version> wireCompat = new ArrayList<>();
+
+        List<Version> prevMajors = groupByMajor.get(currentVersion.getMajor() - 1);
+        int minor = prevMajors.get(prevMajors.size() - 1).getMinor();
+        for (int i = prevMajors.size() - 1;
+             i > 0 && prevMajors.get(i).getMinor() == minor;
+             i--
+        ) {
+            wireCompat.add(prevMajors.get(i));
+        }
+        wireCompat.addAll(groupByMajor.get(currentVersion.getMajor()));
+        wireCompat.remove(currentVersion);
+        wireCompat.sort(Version::compareTo);
+
+        return unmodifiableList(wireCompat);
+    }
+
+    public List<Version> getUnreleasedIndexCompatible() {
+        List<Version> unreleasedIndexCompatible = new ArrayList<>(getIndexCompatible());
+        unreleasedIndexCompatible.retainAll(getUnreleased());
+        return unmodifiableList(unreleasedIndexCompatible);
+    }
+
+    public List<Version> getUnreleasedWireCompatible() {
+        List<Version> unreleasedWireCompatible = new ArrayList<>(getWireCompatible());
+        unreleasedWireCompatible.retainAll(getUnreleased());
+        return unmodifiableList(unreleasedWireCompatible);
+    }
+
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java
index 9ee597eb25ad8..23ac9458b961d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java
@@ -10,7 +10,7 @@
  * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
  */
 public class VersionProperties {
-    public static Version getElasticsearch() {
+    public static String getElasticsearch() {
         return elasticsearch;
     }
 
@@ -22,12 +22,12 @@ public static Map<String, String> getVersions() {
         return versions;
     }
 
-    private static final Version elasticsearch;
+    private static final String elasticsearch;
     private static final String lucene;
     private static final Map<String, String> versions = new HashMap<String, String>();
     static {
         Properties props = getVersionProperties();
-        elasticsearch = Version.fromString(props.getProperty("elasticsearch"));
+        elasticsearch = props.getProperty("elasticsearch");
         lucene = props.getProperty("lucene");
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
@@ -38,13 +38,17 @@ private static Properties getVersionProperties() {
         Properties props = new Properties();
         InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties");
         if (propsStream == null) {
-            throw new RuntimeException("/version.properties resource missing");
+            throw new IllegalStateException("/version.properties resource missing");
         }
         try {
             props.load(propsStream);
         } catch (IOException e) {
-            throw new RuntimeException(e);
+            throw new IllegalStateException("Failed to load version properties", e);
         }
         return props;
     }
+
+    public static boolean isElasticsearchSnapshot() {
+        return elasticsearch.endsWith("-SNAPSHOT");
+    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index 85931c7846b34..012e05f2f6c8d 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -25,16 +25,13 @@
 import org.gradle.api.logging.Logging;
 
 import java.util.Objects;
-import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 
 public class ElasticsearchNode {
 
     private final String name;
     private final GradleServicesAdapter services;
-    private final AtomicInteger noOfClaims = new AtomicInteger();
-    private final AtomicBoolean started = new AtomicBoolean(false);
+    private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);
     private final Logger logger = Logging.getLogger(ElasticsearchNode.class);
 
     private Distribution distribution;
@@ -54,7 +51,7 @@ public Version getVersion() {
     }
 
     public void setVersion(Version version) {
-        checkNotRunning();
+        checkFrozen();
         this.version = version;
     }
 
@@ -63,47 +60,26 @@ public Distribution getDistribution() {
     }
 
     public void setDistribution(Distribution distribution) {
-        checkNotRunning();
+        checkFrozen();
         this.distribution = distribution;
     }
 
-    public void claim() {
-        noOfClaims.incrementAndGet();
+    void start() {
+        logger.info("Starting `{}`", this);
     }
 
-    /**
-     * Start the cluster if not running. Does nothing if the cluster is already running.
-     *
-     * @return future of thread running in the background
-     */
-    public Future<Void> start() {
-        if (started.getAndSet(true)) {
-            logger.lifecycle("Already started cluster: {}", name);
-        } else {
-            logger.lifecycle("Starting cluster: {}", name);
-        }
-        return null;
+    void stop(boolean tailLogs) {
+        logger.info("Stopping `{}`, tailLogs: {}", this, tailLogs);
     }
 
-    /**
-     * Stops a running cluster if it's not claimed. Does nothing otherwise.
-     */
-    public void unClaimAndStop() {
-        int decrementedClaims = noOfClaims.decrementAndGet();
-        if (decrementedClaims > 0) {
-            logger.lifecycle("Not stopping {}, since cluster still has {} claim(s)", name, decrementedClaims);
-            return;
-        }
-        if (started.get() == false) {
-            logger.lifecycle("Asked to unClaimAndStop, but cluster was not running: {}", name);
-            return;
-        }
-        logger.lifecycle("Stopping {}, number of claims is {}", name, decrementedClaims);
+    public void freeze() {
+        logger.info("Locking configuration of `{}`", this);
+        configurationFrozen.set(true);
     }
 
-    private void checkNotRunning() {
-        if (started.get()) {
-            throw new IllegalStateException("Configuration can not be altered while running ");
+    private void checkFrozen() {
+        if (configurationFrozen.get()) {
+            throw new IllegalStateException("Configuration can not be altered, already locked");
         }
     }
 
@@ -119,4 +95,9 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(name);
     }
+
+    @Override
+    public String toString() {
+        return "ElasticsearchNode{name='" + name + "'}";
+    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
index fcd83a1f46101..5191c7d4febb2 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -33,73 +33,171 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 public class TestClustersPlugin implements Plugin<Project> {
 
     private static final String LIST_TASK_NAME = "listTestClusters";
     private static final String NODE_EXTENSION_NAME = "testClusters";
+    public static final String PROPERTY_TESTCLUSTERS_RUN_ONCE = "_testclusters_run_once";
 
     private final Logger logger =  Logging.getLogger(TestClustersPlugin.class);
 
+    // this is static because we need a single mapping across multi project builds, as some of the listeners we use,
+    // like task graph are singletons across multi project builds.
+    private static final Map<Task, List<ElasticsearchNode>> usedClusters = new ConcurrentHashMap<>();
+    private static final Map<ElasticsearchNode, Integer> claimsInventory = new ConcurrentHashMap<>();
+    private static final Set<ElasticsearchNode> runningClusters = Collections.synchronizedSet(new HashSet<>());
+
     @Override
     public void apply(Project project) {
-        NamedDomainObjectContainer<? extends ElasticsearchNode> container = project.container(
+        // enable the DSL to describe clusters
+        NamedDomainObjectContainer<ElasticsearchNode> container = createTestClustersContainerExtension(project);
+
+        // provide a task to be able to list defined clusters.
+        createListClustersTask(project, container);
+
+        // create DSL for tasks to mark clusters these use
+        createUseClusterTaskExtension(project);
+
+        // There's a single Gradle instance for multi project builds, this means that some configuration needs to be
+        // done only once even if the plugin is applied multiple times as a part of multi project build
+        ExtraPropertiesExtension rootProperties = project.getRootProject().getExtensions().getExtraProperties();
+        if (rootProperties.has(PROPERTY_TESTCLUSTERS_RUN_ONCE) == false) {
+            rootProperties.set(PROPERTY_TESTCLUSTERS_RUN_ONCE, true);
+            // When running in the Daemon it's possible for this to hold references to past
+            usedClusters.clear();
+            claimsInventory.clear();
+            runningClusters.clear();
+
+            // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
+            // that are defined in the build script and the ones that will actually be used in this invocation of gradle
+            // we use this information to determine when the last task that required the cluster executed so that we can
+            // terminate the cluster right away and free up resources.
+            configureClaimClustersHook(project);
+
+            // Before each task, we determine if a cluster needs to be started for that task.
+            configureStartClustersHook(project);
+
+            // After each task we determine if there are clusters that are no longer needed.
+            configureStopClustersHook(project);
+        }
+    }
+
+    private NamedDomainObjectContainer<ElasticsearchNode> createTestClustersContainerExtension(Project project) {
+        // Create an extensions that allows describing clusters
+        NamedDomainObjectContainer<ElasticsearchNode> container = project.container(
             ElasticsearchNode.class,
-            (name) -> new ElasticsearchNode(name, GradleServicesAdapter.getInstance(project))
+            name -> new ElasticsearchNode(
+                name,
+                GradleServicesAdapter.getInstance(project)
+            )
         );
         project.getExtensions().add(NODE_EXTENSION_NAME, container);
+        return container;
+    }
+
 
+    private void createListClustersTask(Project project, NamedDomainObjectContainer<ElasticsearchNode> container) {
         Task listTask = project.getTasks().create(LIST_TASK_NAME);
         listTask.setGroup("ES cluster formation");
         listTask.setDescription("Lists all ES clusters configured for this project");
         listTask.doLast((Task task) ->
-            container.forEach((ElasticsearchNode cluster) ->
+            container.forEach(cluster ->
                 logger.lifecycle("   * {}: {}", cluster.getName(), cluster.getDistribution())
             )
         );
+    }
 
-        Map<Task, List<ElasticsearchNode>> taskToCluster = new HashMap<>();
-
+    private void createUseClusterTaskExtension(Project project) {
         // register an extension for all current and future tasks, so that any task can declare that it wants to use a
         // specific cluster.
         project.getTasks().all((Task task) ->
             task.getExtensions().findByType(ExtraPropertiesExtension.class)
-            .set(
-                "useCluster",
-                new Closure<Void>(this, this) {
-                    public void doCall(ElasticsearchNode conf) {
-                        taskToCluster.computeIfAbsent(task, k -> new ArrayList<>()).add(conf);
-                    }
-                })
+                .set(
+                    "useCluster",
+                    new Closure<Void>(this, task) {
+                        public void doCall(ElasticsearchNode node) {
+                            usedClusters.computeIfAbsent(task, k -> new ArrayList<>()).add(node);
+                        }
+                    })
         );
+    }
 
+    private void configureClaimClustersHook(Project project) {
         project.getGradle().getTaskGraph().whenReady(taskExecutionGraph ->
             taskExecutionGraph.getAllTasks()
                 .forEach(task ->
-                    taskToCluster.getOrDefault(task, Collections.emptyList()).forEach(ElasticsearchNode::claim)
+                    usedClusters.getOrDefault(task, Collections.emptyList()).forEach(each -> {
+                        synchronized (claimsInventory) {
+                            claimsInventory.put(each, claimsInventory.getOrDefault(each, 0) + 1);
+                        }
+                        each.freeze();
+                    })
                 )
         );
+    }
+
+    private void configureStartClustersHook(Project project) {
         project.getGradle().addListener(
             new TaskActionListener() {
                 @Override
                 public void beforeActions(Task task) {
                     // we only start the cluster before the actions, so we'll not start it if the task is up-to-date
-                    taskToCluster.getOrDefault(task, new ArrayList<>()).forEach(ElasticsearchNode::start);
+                    final List<ElasticsearchNode> clustersToStart;
+                    synchronized (runningClusters) {
+                        clustersToStart = usedClusters.getOrDefault(task,Collections.emptyList()).stream()
+                            .filter(each -> runningClusters.contains(each) == false)
+                            .collect(Collectors.toList());
+                        runningClusters.addAll(clustersToStart);
+                    }
+                    clustersToStart.forEach(ElasticsearchNode::start);
+
                 }
                 @Override
                 public void afterActions(Task task) {}
             }
         );
+    }
+
+    private void configureStopClustersHook(Project project) {
         project.getGradle().addListener(
             new TaskExecutionListener() {
                 @Override
                 public void afterExecute(Task task, TaskState state) {
-                    // always un-claim the cluster, even if _this_ task is up-to-date, as others might not have been and caused the
-                    // cluster to start.
-                    taskToCluster.getOrDefault(task, new ArrayList<>()).forEach(ElasticsearchNode::unClaimAndStop);
+                    // always unclaim the cluster, even if _this_ task is up-to-date, as others might not have been
+                    // and caused the cluster to start.
+                    List<ElasticsearchNode> clustersUsedByTask = usedClusters.getOrDefault(
+                        task,
+                        Collections.emptyList()
+                    );
+                    if (state.getFailure() != null) {
+                        // If the task fails, and other tasks use this cluster, the other task will likely never be
+                        // executed at all, so we will never get to un-claim and terminate it.
+                        // The downside is that with multi project builds if that other  task is in a different
+                        // project and executing right now, we may terminate the cluster while it's running it.
+                        clustersUsedByTask.forEach(each -> each.stop(true));
+                    } else {
+                        clustersUsedByTask.forEach(each -> {
+                            synchronized (claimsInventory) {
+                                claimsInventory.put(each, claimsInventory.get(each) - 1);
+                            }
+                        });
+                        final List<ElasticsearchNode> stoppable;
+                        synchronized (runningClusters) {
+                            stoppable = claimsInventory.entrySet().stream()
+                                .filter(entry -> entry.getValue() == 0)
+                                .filter(entry -> runningClusters.contains(entry.getKey()))
+                                .map(Map.Entry::getKey)
+                                .collect(Collectors.toList());
+                        }
+                        stoppable.forEach(each -> each.stop(false));
+                    }
                 }
                 @Override
                 public void beforeExecute(Task task) {}
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 5fb10036a09aa..6a07c9058c836 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -82,20 +82,6 @@
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksRequestBuilder.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]TransportPendingClusterTasksAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BackoffPolicy.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkProcessor.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkRequest.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkResponse.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]Retry.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportBulkAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportShardBulkAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]delete[/\\]DeleteRequest.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]explain[/\\]TransportExplainAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]GetRequest.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetRequest.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportGetAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportShardMultiGetAction.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]IndexRequest.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestBuilder.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseController.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]ShardSearchFailure.java" checks="LineLength" />
@@ -142,29 +128,6 @@
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngine.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]LiveVersionMap.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]get[/\\]ShardGetService.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMappers.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParser.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicTemplate.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeLookup.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoShapeFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IdFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IndexFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MappedFieldType.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperService.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapping.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MetadataFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ObjectMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ParentFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RootObjectMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RoutingFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]SourceFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]TypeFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UidFieldMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]VersionFieldMapper.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]merge[/\\]MergeStats.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]AbstractQueryBuilder.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryBuilders.java" checks="LineLength" />
@@ -267,9 +230,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]SnapshotBlocksIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]state[/\\]ClusterStateRequestTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]ClusterStatsIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkRequestTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]RetryTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetShardRequestTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchRequestBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]TransportActionFilterChainTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]WaitActiveShardCountIT.java" checks="LineLength" />
@@ -293,33 +253,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltAnalyzerTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngineMergeIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]InternalEngineTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CompletionFieldTypeTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]CopyToMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMetadataMapper.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoPointFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]GeoShapeFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IdFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]IndexFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]JavaMultiFieldMergeTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperServiceTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MultiFieldCopyToMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MultiFieldTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]NestedObjectMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]NullValueObjectMappingTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ObjectMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]PathMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ParentFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]RoutingFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]SourceFieldMapperTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]UpdateMappingOnClusterIT.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoolQueryBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoostingQueryBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]GeoDistanceQueryBuilderTests.java" checks="LineLength" />
diff --git a/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTests.groovy b/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTests.groovy
deleted file mode 100644
index ad36c84078398..0000000000000
--- a/buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTests.groovy
+++ /dev/null
@@ -1,236 +0,0 @@
-package org.elasticsearch.gradle
-
-import org.elasticsearch.gradle.test.GradleUnitTestCase
-import org.junit.Test
-
-class VersionCollectionTests extends GradleUnitTestCase {
-
-  String formatVersion(String version) {
-    return " public static final Version V_${version.replaceAll("\\.", "_")} "
-  }
-  List<String> allVersions = [formatVersion('5.0.0'), formatVersion('5.0.0_alpha1'), formatVersion('5.0.0_alpha2'), formatVersion('5.0.0_beta1'),
-                     formatVersion('5.0.0_rc1'),formatVersion('5.0.0_rc2'),formatVersion('5.0.1'), formatVersion('5.0.2'),
-                     formatVersion('5.1.1'), formatVersion('5.1.2'), formatVersion('5.2.0'), formatVersion('5.2.1'), formatVersion('6.0.0'),
-                     formatVersion('6.0.1'), formatVersion('6.1.0'), formatVersion('6.1.1'), formatVersion('6.2.0'), formatVersion('6.3.0'),
-                     formatVersion('7.0.0_alpha1'), formatVersion('7.0.0_alpha2')]
-
-  /**
-   * This validates the logic of being on a unreleased major branch with a staged major-1.minor sibling. This case happens when a version is
-   * branched from Major-1.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
-   * from 7.0 perspective, or master at the time of this writing.
-   */
-  @Test
-  void testAgainstMajorUnreleasedWithExistingStagedMinorRelease() {
-    VersionCollection vc = new VersionCollection(allVersions)
-    assertNotNull(vc)
-    assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT"))
-    assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT"))
-    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT"))
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
-
-    vc.indexCompatible.containsAll(vc.versions)
-
-    // This should contain the same list sans the current version
-    List indexCompatList =  [Version.fromString("6.0.0"), Version.fromString("6.0.1"),
-                             Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"),
-                             Version.fromString("6.2.0-SNAPSHOT"), Version.fromString("6.3.0-SNAPSHOT")]
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")]
-    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 3)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
-
-    assertEquals(vc.snapshotsWireCompatible.size(), 1)
-    assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT"))
-  }
-
-  /**
-   * This validates the logic of being on a unreleased major branch without a staged major-1.minor sibling. This case happens once a staged,
-   * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
-   * simulates the behavior from 7.0 perspective, or master at the time of this writing.
-   */
-  @Test
-  void testAgainstMajorUnreleasedWithoutStagedMinorRelease() {
-    List localVersion = allVersions.clone()
-    localVersion.add(formatVersion('6.2.1')) // release 6.2
-
-    VersionCollection vc = new VersionCollection(localVersion)
-    assertNotNull(vc)
-    assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT"))
-    assertEquals(vc.stagedMinorSnapshot, null)
-    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT"))
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
-
-    vc.indexCompatible.containsAll(vc.versions)
-
-    // This should contain the same list sans the current version
-    List indexCompatList =  [Version.fromString("6.0.0"), Version.fromString("6.0.1"),
-                             Version.fromString("6.1.0"), Version.fromString("6.1.1"),
-                             Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT"),
-                             Version.fromString("6.3.0-SNAPSHOT")]
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")]
-    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 2)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
-
-    assertEquals(vc.snapshotsWireCompatible.size(), 1)
-    assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT"))
-  }
-
-  /**
-   * This validates the logic of being on a unreleased minor branch with a staged minor sibling. This case happens when a version is
-   * branched from Major.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
-   * from 6.3 perspective.
-   */
-  @Test
-  void testAgainstMinorReleasedBranch() {
-    List localVersion = allVersions.clone()
-    localVersion.removeAll { it.toString().contains('7_0_0')} // remove all the 7.x so that the actual version is 6.3 (6.x)
-    VersionCollection vc = new VersionCollection(localVersion)
-    assertNotNull(vc)
-    assertEquals(vc.nextMinorSnapshot, null)
-    assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT"))
-    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT"))
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
-
-    // This should contain the same list sans the current version
-    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
-                           Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"),
-                           Version.fromString("6.2.0-SNAPSHOT")]
-    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 3)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-
-    assertEquals(vc.snapshotsWireCompatible.size(), 3)
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-  }
-
-  /**
-   * This validates the logic of being on a unreleased minor branch without a staged minor sibling. This case happens once a staged,
-   * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
-   * simulates the behavior from 6.3 perspective.
-   */
-  @Test
-  void testAgainstMinorReleasedBranchNoStagedMinor() {
-    List localVersion = allVersions.clone()
-    // remove all the 7.x and add a 6.2.1 which means 6.2 was released
-    localVersion.removeAll { it.toString().contains('7_0_0')}
-    localVersion.add(formatVersion('6.2.1'))
-    VersionCollection vc = new VersionCollection(localVersion)
-    assertNotNull(vc)
-    assertEquals(vc.nextMinorSnapshot, null)
-    assertEquals(vc.stagedMinorSnapshot, null)
-    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT"))
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
-
-    // This should contain the same list sans the current version
-    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
-                           Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1"),
-                           Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT")]
-    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 2)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-
-    assertEquals(vc.snapshotsWireCompatible.size(), 2)
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-  }
-
-  /**
-   * This validates the logic of being on a released minor branch. At the time of writing, 6.2 is unreleased, so this is equivalent of being
-   * on 6.1.
-   */
-  @Test
-  void testAgainstOldMinor() {
-
-    List localVersion = allVersions.clone()
-    // remove the 7 alphas and the ones greater than 6.1
-    localVersion.removeAll { it.toString().contains('7_0_0') || it.toString().contains('V_6_2') || it.toString().contains('V_6_3') }
-    VersionCollection vc = new VersionCollection(localVersion)
-    assertNotNull(vc)
-    assertEquals(vc.nextMinorSnapshot, null)
-    assertEquals(vc.stagedMinorSnapshot, null)
-    assertEquals(vc.nextBugfixSnapshot, null)
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
-
-    // This should contain the same list sans the current version
-    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
-                           Version.fromString("6.0.1"), Version.fromString("6.1.0")]
-    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 1)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-
-    assertEquals(vc.snapshotsWireCompatible.size(), 1)
-    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
-  }
-
-  /**
-   * This validates the lower bound of wire compat, which is 5.0. It also validates that the span of 2.x to 5.x if it is decided to port
-   * this fix all the way to the maint 5.6 release.
-   */
-  @Test
-  void testFloorOfWireCompatVersions() {
-    List localVersion = [formatVersion('2.0.0'), formatVersion('2.0.1'), formatVersion('2.1.0'), formatVersion('2.1.1'),
-                          formatVersion('5.0.0'), formatVersion('5.0.1'), formatVersion('5.1.0'), formatVersion('5.1.1'),
-                          formatVersion('5.2.0'),formatVersion('5.2.1'),formatVersion('5.3.0'),formatVersion('5.3.1'),
-                          formatVersion('5.3.2')]
-    VersionCollection vc = new VersionCollection(localVersion)
-    assertNotNull(vc)
-    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("2.1.1-SNAPSHOT"))
-
-    // This should contain the same list sans the current version
-    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
-    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
-    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
-
-    List wireCompatList = [Version.fromString("2.1.0"), Version.fromString("2.1.1-SNAPSHOT"), Version.fromString("5.0.0"),
-                           Version.fromString("5.0.1"), Version.fromString("5.1.0"),
-                           Version.fromString("5.1.1"), Version.fromString("5.2.0"), Version.fromString("5.2.1"),
-                           Version.fromString("5.3.0"), Version.fromString("5.3.1")]
-
-    List<Version> compatible = vc.wireCompatible
-    assertTrue(wireCompatList.containsAll(compatible))
-    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
-
-    assertEquals(vc.snapshotsIndexCompatible.size(), 1)
-    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("2.1.1-SNAPSHOT")))
-
-    // ensure none of the 2.x snapshots appear here, as this is the floor of bwc for wire compat
-    assertEquals(vc.snapshotsWireCompatible.size(), 0)
-  }
-}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
index 2a2304182c80e..661dba7716a7f 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.GradleRunner;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.rules.TemporaryFolder;
 
@@ -38,6 +39,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+@Ignore
 public class BuildExamplePluginsIT extends GradleIntegrationTestCase {
 
     private static List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java
new file mode 100644
index 0000000000000..d1b4e893ec6ad
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java
@@ -0,0 +1,406 @@
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class VersionCollectionTests extends GradleUnitTestCase {
+
+    private static final Map<String, List<String>> sampleVersions = new HashMap<>();
+
+    @Rule
+    public ExpectedException expectedEx = ExpectedException.none();
+
+    static {
+        // unreleased major and two unreleased minors ( minor in feature freeze )
+        sampleVersions.put("8.0.0", asList(
+            "7_0_0", "7_0_1", "7_1_0", "7_1_1", "7_2_0", "7_3_0", "8.0.0"
+        ));
+        sampleVersions.put("7.0.0-alpha1", asList(
+            "6_0_0_alpha1", "6_0_0_alpha2", "6_0_0_beta1", "6_0_0_beta2", "6_0_0_rc1", "6_0_0_rc2",
+            "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4",
+            "6_2_0", "6_2_1", "6_2_2", "6_2_3", "6_2_4",
+            "6_3_0", "6_3_1", "6_3_2",
+            "6_4_0", "6_4_1", "6_4_2",
+            "6_5_0", "7_0_0_alpha1"
+        ));
+        sampleVersions.put("6.5.0", asList(
+            "5_0_0_alpha1", "5_0_0_alpha2", "5_0_0_alpha3", "5_0_0_alpha4", "5_0_0_alpha5", "5_0_0_beta1", "5_0_0_rc1",
+            "5_0_0", "5_0_1", "5_0_2", "5_1_1", "5_1_2", "5_2_0", "5_2_1", "5_2_2", "5_3_0", "5_3_1", "5_3_2", "5_3_3",
+            "5_4_0", "5_4_1", "5_4_2", "5_4_3", "5_5_0", "5_5_1", "5_5_2", "5_5_3", "5_6_0", "5_6_1", "5_6_2", "5_6_3",
+            "5_6_4", "5_6_5", "5_6_6", "5_6_7", "5_6_8", "5_6_9", "5_6_10", "5_6_11", "5_6_12", "5_6_13",
+            "6_0_0_alpha1", "6_0_0_alpha2", "6_0_0_beta1", "6_0_0_beta2", "6_0_0_rc1", "6_0_0_rc2", "6_0_0", "6_0_1",
+            "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3", "6_2_4", "6_3_0", "6_3_1",
+            "6_3_2", "6_4_0", "6_4_1", "6_4_2", "6_5_0"
+        ));
+        sampleVersions.put("6.6.0", asList(
+            "5_0_0_alpha1", "5_0_0_alpha2", "5_0_0_alpha3", "5_0_0_alpha4", "5_0_0_alpha5", "5_0_0_beta1", "5_0_0_rc1",
+            "5_0_0", "5_0_1", "5_0_2", "5_1_1", "5_1_2", "5_2_0", "5_2_1", "5_2_2", "5_3_0", "5_3_1", "5_3_2", "5_3_3",
+            "5_4_0", "5_4_1", "5_4_2", "5_4_3", "5_5_0", "5_5_1", "5_5_2", "5_5_3", "5_6_0", "5_6_1", "5_6_2", "5_6_3",
+            "5_6_4", "5_6_5", "5_6_6", "5_6_7", "5_6_8", "5_6_9", "5_6_10", "5_6_11", "5_6_12", "5_6_13",
+            "6_0_0_alpha1", "6_0_0_alpha2", "6_0_0_beta1", "6_0_0_beta2", "6_0_0_rc1", "6_0_0_rc2", "6_0_0", "6_0_1",
+            "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3", "6_2_4", "6_3_0", "6_3_1",
+            "6_3_2", "6_4_0", "6_4_1", "6_4_2", "6_5_0", "6_6_0"
+        ));
+        sampleVersions.put("6.4.2", asList(
+            "5_0_0_alpha1", "5_0_0_alpha2", "5_0_0_alpha3", "5_0_0_alpha4", "5_0_0_alpha5", "5_0_0_beta1", "5_0_0_rc1",
+            "5_0_0", "5_0_1", "5_0_2", "5_1_1", "5_1_2", "5_2_0", "5_2_1", "5_2_2", "5_3_0",
+            "5_3_1", "5_3_2", "5_3_3", "5_4_0", "5_4_1", "5_4_2", "5_4_3", "5_5_0", "5_5_1", "5_5_2", "5_5_3",
+            "5_6_0", "5_6_1", "5_6_2", "5_6_3", "5_6_4", "5_6_5", "5_6_6", "5_6_7", "5_6_8", "5_6_9", "5_6_10",
+            "5_6_11", "5_6_12", "5_6_13",
+            "6_0_0_alpha1", "6_0_0_alpha2", "6_0_0_beta1", "6_0_0_beta2", "6_0_0_rc1", "6_0_0_rc2",
+            "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3",
+            "6_2_4", "6_3_0", "6_3_1", "6_3_2", "6_4_0", "6_4_1", "6_4_2"
+        ));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testExceptionOnEmpty() {
+        new VersionCollection(asList("foo", "bar"), Version.fromString("7.0.0"));
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testExceptionOnNonCurrent() {
+        new VersionCollection(singletonList(formatVersionToLine("6.5.0")), Version.fromString("7.0.0"));
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testExceptionOnTooManyMajors() {
+        new VersionCollection(
+            asList(
+                formatVersionToLine("5.6.12"),
+                formatVersionToLine("6.5.0"),
+                formatVersionToLine("7.0.0")
+            ),
+            Version.fromString("6.5.0")
+        );
+    }
+
+    public void testWireCompatible() {
+        assertVersionsEquals(
+            singletonList("6.5.0-SNAPSHOT"),
+            getVersionCollection("7.0.0-alpha1").getWireCompatible()
+        );
+        assertVersionsEquals(
+            asList(
+                "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
+                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
+                "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT"
+            ),
+            getVersionCollection("6.5.0").getWireCompatible()
+        );
+
+        assertVersionsEquals(
+            asList(
+                "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
+                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT", "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
+                "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4", "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1"
+            ),
+            getVersionCollection("6.4.2").getWireCompatible()
+        );
+
+        assertVersionsEquals(
+            asList(
+                "5.6.0", "5.6.1", "5.6.2", "5.6.3", "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10",
+                "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
+                "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+            ),
+            getVersionCollection("6.6.0").getWireCompatible()
+        );
+
+        assertVersionsEquals(
+            singletonList("7.3.0"),
+            getVersionCollection("8.0.0").getWireCompatible()
+        );
+    }
+
+    public void testWireCompatibleUnreleased() {
+        assertVersionsEquals(
+            singletonList("6.5.0-SNAPSHOT"),
+            getVersionCollection("7.0.0-alpha1").getUnreleasedWireCompatible()
+        );
+        assertVersionsEquals(
+            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT"),
+            getVersionCollection("6.5.0").getUnreleasedWireCompatible()
+        );
+
+        assertVersionsEquals(
+            singletonList("5.6.13-SNAPSHOT"),
+            getVersionCollection("6.4.2").getUnreleasedWireCompatible()
+        );
+
+        assertVersionsEquals(
+            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            getVersionCollection("6.6.0").getUnreleasedWireCompatible()
+        );
+
+        assertVersionsEquals(
+            singletonList("7.3.0"),
+            getVersionCollection("8.0.0").getUnreleasedWireCompatible()
+        );
+    }
+
+    public void testIndexCompatible() {
+        assertVersionsEquals(
+            asList(
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4",
+                "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4", "6.3.0", "6.3.1",
+                "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+            ),
+            getVersionCollection("7.0.0-alpha1").getIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList(
+                "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
+                "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT"
+            ),
+            getVersionCollection("6.5.0").getIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList(
+                "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
+                "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1"
+            ),
+            getVersionCollection("6.4.2").getIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList(
+                "5.0.0", "5.0.1", "5.0.2", "5.1.1", "5.1.2", "5.2.0", "5.2.1", "5.2.2", "5.3.0", "5.3.1", "5.3.2", "5.3.3",
+                "5.4.0", "5.4.1", "5.4.2", "5.4.3", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.6.0", "5.6.1", "5.6.2", "5.6.3",
+                "5.6.4", "5.6.5", "5.6.6", "5.6.7", "5.6.8", "5.6.9", "5.6.10", "5.6.11", "5.6.12", "5.6.13-SNAPSHOT",
+                "6.0.0", "6.0.1", "6.1.0", "6.1.1", "6.1.2", "6.1.3", "6.1.4", "6.2.0", "6.2.1", "6.2.2", "6.2.3", "6.2.4",
+                "6.3.0", "6.3.1", "6.3.2", "6.4.0", "6.4.1", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"
+            ),
+            getVersionCollection("6.6.0").getIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList("7.0.0", "7.0.1", "7.1.0", "7.1.1", "7.2.0", "7.3.0"),
+            getVersionCollection("8.0.0").getIndexCompatible()
+        );
+    }
+
+    public void testIndexCompatibleUnreleased() {
+        assertVersionsEquals(
+            asList("6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            getVersionCollection("7.0.0-alpha1").getUnreleasedIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT"),
+            getVersionCollection("6.5.0").getUnreleasedIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            singletonList("5.6.13-SNAPSHOT"),
+            getVersionCollection("6.4.2").getUnreleasedIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList("5.6.13-SNAPSHOT", "6.4.2-SNAPSHOT", "6.5.0-SNAPSHOT"),
+            getVersionCollection("6.6.0").getUnreleasedIndexCompatible()
+        );
+
+        assertVersionsEquals(
+            asList("7.1.1", "7.2.0", "7.3.0"),
+            getVersionCollection("8.0.0").getUnreleasedIndexCompatible()
+        );
+    }
+
+    public void testGetUnreleased() {
+        assertVersionsEquals(
+            asList("6.4.2", "6.5.0", "7.0.0-alpha1"),
+            getVersionCollection("7.0.0-alpha1").getUnreleased()
+        );
+        assertVersionsEquals(
+            asList("5.6.13", "6.4.2", "6.5.0"),
+            getVersionCollection("6.5.0").getUnreleased()
+        );
+        assertVersionsEquals(
+            asList("5.6.13", "6.4.2"),
+            getVersionCollection("6.4.2").getUnreleased()
+        );
+        assertVersionsEquals(
+            asList("5.6.13", "6.4.2", "6.5.0", "6.6.0"),
+            getVersionCollection("6.6.0").getUnreleased()
+        );
+        assertVersionsEquals(
+            asList("7.1.1", "7.2.0", "7.3.0", "8.0.0"),
+            getVersionCollection("8.0.0").getUnreleased()
+        );
+    }
+
+    public void testGetBranch() {
+        assertUnreleasedBranchNames(
+            asList("6.4", "6.x"),
+            getVersionCollection("7.0.0-alpha1")
+        );
+        assertUnreleasedBranchNames(
+            asList("5.6", "6.4"),
+            getVersionCollection("6.5.0")
+        );
+        assertUnreleasedBranchNames(
+            singletonList("5.6"),
+            getVersionCollection("6.4.2")
+        );
+        assertUnreleasedBranchNames(
+            asList("5.6", "6.4", "6.5"),
+            getVersionCollection("6.6.0")
+        );
+        assertUnreleasedBranchNames(
+            asList("7.1", "7.2", "7.x"),
+            getVersionCollection("8.0.0")
+        );
+    }
+
+    public void testGetGradleProjectName() {
+        assertUnreleasedGradleProjectNames(
+            asList("bugfix", "minor"),
+            getVersionCollection("7.0.0-alpha1")
+        );
+        assertUnreleasedGradleProjectNames(
+            asList("maintenance", "bugfix"),
+            getVersionCollection("6.5.0")
+        );
+        assertUnreleasedGradleProjectNames(
+            singletonList("maintenance"),
+            getVersionCollection("6.4.2")
+        );
+        assertUnreleasedGradleProjectNames(
+            asList("maintenance", "bugfix", "staged"),
+            getVersionCollection("6.6.0")
+        );
+        assertUnreleasedGradleProjectNames(
+            asList("bugfix", "staged", "minor"),
+            getVersionCollection("8.0.0")
+        );
+    }
+
+    public void testCompareToAuthoritative() {
+        List<String> listOfVersions = asList("7.0.0", "7.0.1", "7.1.0", "7.1.1", "7.2.0", "7.3.0", "8.0.0");
+        List<Version> authoritativeReleasedVersions = Stream.of("7.0.0", "7.0.1", "7.1.0")
+                .map(Version::fromString)
+                .collect(Collectors.toList());
+
+        VersionCollection vc = new VersionCollection(
+            listOfVersions.stream()
+                .map(this::formatVersionToLine)
+                .collect(Collectors.toList()),
+            Version.fromString("8.0.0")
+        );
+        vc.compareToAuthoritative(authoritativeReleasedVersions);
+    }
+
+    public void testCompareToAuthoritativeUnreleasedActuallyReleased() {
+        List<String> listOfVersions = asList("7.0.0", "7.0.1", "7.1.0", "7.1.1", "7.2.0", "7.3.0", "8.0.0");
+        List<Version> authoritativeReleasedVersions = Stream.of("7.0.0", "7.0.1", "7.1.0", "7.1.1", "8.0.0")
+                .map(Version::fromString)
+                .collect(Collectors.toList());
+
+        VersionCollection vc = new VersionCollection(
+            listOfVersions.stream()
+                .map(this::formatVersionToLine)
+                .collect(Collectors.toList()),
+            Version.fromString("8.0.0")
+        );
+        expectedEx.expect(IllegalStateException.class);
+        expectedEx.expectMessage("but they are released");
+        vc.compareToAuthoritative(authoritativeReleasedVersions);
+    }
+
+    public void testCompareToAuthoritativeNotReallyRelesed() {
+        List<String> listOfVersions = asList("7.0.0", "7.0.1", "7.1.0", "7.1.1", "7.2.0", "7.3.0", "8.0.0");
+        List<Version> authoritativeReleasedVersions = Stream.of("7.0.0", "7.0.1")
+                .map(Version::fromString)
+                .collect(Collectors.toList());
+        VersionCollection vc = new VersionCollection(
+            listOfVersions.stream()
+                .map(this::formatVersionToLine)
+                .collect(Collectors.toList()),
+            Version.fromString("8.0.0")
+        );
+        expectedEx.expect(IllegalStateException.class);
+        expectedEx.expectMessage("not really released");
+        vc.compareToAuthoritative(authoritativeReleasedVersions);
+    }
+
+    private void assertUnreleasedGradleProjectNames(List<String> expectedNAmes, VersionCollection versionCollection) {
+        List<String> actualNames = new ArrayList<>();
+        versionCollection.forPreviousUnreleased(unreleasedVersion ->
+            actualNames.add(unreleasedVersion.gradleProjectName)
+        );
+        assertEquals(expectedNAmes, actualNames);
+    }
+
+    private void assertUnreleasedBranchNames(List<String> expectedBranches, VersionCollection versionCollection) {
+        List<String> actualBranches = new ArrayList<>();
+        versionCollection.forPreviousUnreleased(unreleasedVersionInfo ->
+            actualBranches.add(unreleasedVersionInfo.branch)
+        );
+        assertEquals(expectedBranches, actualBranches);
+    }
+
+    private String formatVersionToLine(final String version) {
+        return " public static final Version V_" + version.replaceAll("\\.", "_") + " ";
+    }
+
+    private void assertVersionsEquals(List<String> expected, List<Version> actual) {
+        assertEquals(
+            expected.stream()
+                .map(Version::fromString)
+                .collect(Collectors.toList()),
+            actual
+        );
+    }
+
+    private VersionCollection getVersionCollection(String currentVersion) {
+        return new VersionCollection(
+            sampleVersions.get(currentVersion).stream()
+                .map(this::formatVersionToLine)
+                .collect(Collectors.toList()),
+            Version.fromString(currentVersion)
+        );
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java
index d3c3b4a43cb41..3394285157e17 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java
@@ -33,27 +33,23 @@ public class VersionTests extends GradleUnitTestCase {
     public ExpectedException expectedEx = ExpectedException.none();
 
     public void testVersionParsing() {
-        assertVersionEquals("7.0.1", 7, 0, 1, "", false);
-        assertVersionEquals("7.0.1-alpha2", 7, 0, 1, "-alpha2", false);
-        assertVersionEquals("5.1.2-rc3", 5, 1, 2, "-rc3", false);
-        assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2, "", true);
-        assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2, "-beta1", true);
+        assertVersionEquals("7.0.1", 7, 0, 1);
+        assertVersionEquals("7.0.1-alpha2", 7, 0, 1);
+        assertVersionEquals("5.1.2-rc3", 5, 1, 2);
+        assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2);
+        assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2);
     }
 
     public void testCompareWithStringVersions() {
         assertTrue("1.10.20 is not interpreted as before 2.0.0",
             Version.fromString("1.10.20").before("2.0.0")
         );
-        assertTrue("7.0.0-alpha1 is not interpreted as before 7.0.0-alpha2",
-            Version.fromString("7.0.0-alpha1").before("7.0.0-alpha2")
-        );
         assertTrue("7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))
         );
         assertTrue("7.0.0-SNAPSHOT should be equal to 7.0.0-SNAPSHOT",
             Version.fromString("7.0.0-SNAPSHOT").equals(Version.fromString("7.0.0-SNAPSHOT"))
         );
-        assertEquals(Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("5.2.1-SNAPSHOT"));
     }
 
     public void testCollections() {
@@ -78,62 +74,12 @@ public void testCollections() {
     }
 
     public void testToString() {
-        assertEquals("7.0.1", new Version(7, 0, 1, null, false).toString());
+        assertEquals("7.0.1", new Version(7, 0, 1).toString());
     }
 
     public void testCompareVersions() {
-        assertEquals(0, new Version(7, 0, 0, null, true).compareTo(
-            new Version(7, 0, 0, null, true)
-        ));
-        assertEquals(0, new Version(7, 0, 0, null, true).compareTo(
-            new Version(7, 0, 0, "", true)
-        ));
-
-        // snapshot is not taken into account TODO inconsistent with equals
-        assertEquals(
-            0,
-            new Version(7, 0, 0, "", false).compareTo(
-            new Version(7, 0, 0, null, true))
-        );
-        // without sufix is smaller than with TODO
-        assertOrder(
-            new Version(7, 0, 0, null, false),
-            new Version(7, 0, 0, "-alpha1", false)
-        );
-        // numbered sufix
-        assertOrder(
-            new Version(7, 0, 0, "-alpha1", false),
-            new Version(7, 0, 0, "-alpha2", false)
-        );
-        // ranked sufix
-        assertOrder(
-            new Version(7, 0, 0, "-alpha8", false),
-            new Version(7, 0, 0, "-rc1", false)
-        );
-        // ranked sufix
-        assertOrder(
-            new Version(7, 0, 0, "-alpha8", false),
-            new Version(7, 0, 0, "-beta1", false)
-        );
-        // ranked sufix
-        assertOrder(
-            new Version(7, 0, 0, "-beta8", false),
-            new Version(7, 0, 0, "-rc1", false)
-        );
-        // major takes precedence
-        assertOrder(
-            new Version(6, 10, 10, "-alpha8", true),
-            new Version(7, 0, 0, "-alpha2", false)
-        );
-        // then minor
-        assertOrder(
-            new Version(7, 0, 10, "-alpha8", true),
-            new Version(7, 1, 0, "-alpha2", false)
-        );
-        // then revision
-        assertOrder(
-            new Version(7, 1, 0, "-alpha8", true),
-            new Version(7, 1, 10, "-alpha2", false)
+        assertEquals(0,
+            new Version(7, 0, 0).compareTo(new Version(7, 0, 0))
         );
     }
 
@@ -149,33 +95,15 @@ public void testExceptionSyntax() {
         Version.fromString("foo.bar.baz");
     }
 
-    public void testExceptionSuffixNumber() {
-        expectedEx.expect(IllegalArgumentException.class);
-        expectedEx.expectMessage("Invalid suffix");
-        new Version(7, 1, 1, "-alpha", true);
-    }
-
-    public void testExceptionSuffix() {
-        expectedEx.expect(IllegalArgumentException.class);
-        expectedEx.expectMessage("Suffix must contain one of:");
-        new Version(7, 1, 1, "foo1", true);
-    }
-
     private void assertOrder(Version smaller, Version bigger) {
         assertEquals(smaller + " should be smaller than " + bigger, -1, smaller.compareTo(bigger));
     }
 
-    private void assertVersionEquals(String stringVersion, int major, int minor, int revision, String sufix, boolean snapshot) {
+    private void assertVersionEquals(String stringVersion, int major, int minor, int revision) {
         Version version = Version.fromString(stringVersion);
         assertEquals(major, version.getMajor());
         assertEquals(minor, version.getMinor());
         assertEquals(revision, version.getRevision());
-        if (snapshot) {
-            assertTrue("Expected version to be a snapshot but it was not", version.isSnapshot());
-        } else {
-            assertFalse("Expected version not to be a snapshot but it was", version.isSnapshot());
-        }
-        assertEquals(sufix, version.getSuffix());
     }
 
 }
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
index f8e3cf88c4094..025c549489afa 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
@@ -78,8 +78,16 @@ protected void assertTaskFailed(BuildResult result, String taskName) {
         assertTaskOutcome(result, taskName, TaskOutcome.FAILED);
     }
 
-    protected void assertTaskSuccessful(BuildResult result, String taskName) {
-        assertTaskOutcome(result, taskName, TaskOutcome.SUCCESS);
+    protected void assertTaskSuccessful(BuildResult result, String... taskNames) {
+        for (String taskName : taskNames) {
+            assertTaskOutcome(result, taskName, TaskOutcome.SUCCESS);
+        }
+    }
+
+    protected void assertTaskSkipped(BuildResult result, String... taskNames) {
+        for (String taskName : taskNames) {
+            assertTaskOutcome(result, taskName, TaskOutcome.SKIPPED);
+        }
     }
 
     private void assertTaskOutcome(BuildResult result, String taskName, TaskOutcome taskOutcome) {
@@ -96,17 +104,19 @@ private void assertTaskOutcome(BuildResult result, String taskName, TaskOutcome
         );
     }
 
-    protected void assertTaskUpToDate(BuildResult result, String taskName) {
-        BuildTask task = result.task(taskName);
-        if (task == null) {
-            fail("Expected task `" + taskName + "` to be up-to-date, but it did not run");
+    protected void assertTaskUpToDate(BuildResult result, String... taskNames) {
+        for (String taskName : taskNames) {
+            BuildTask task = result.task(taskName);
+            if (task == null) {
+                fail("Expected task `" + taskName + "` to be up-to-date, but it did not run");
+            }
+            assertEquals(
+                "Expected task to be up to date but it was: " + task.getOutcome() +
+                    "\n\nOutput is:\n" + result.getOutput(),
+                TaskOutcome.UP_TO_DATE,
+                task.getOutcome()
+            );
         }
-        assertEquals(
-            "Expected task to be up to date but it was: " + task.getOutcome() +
-                "\n\nOutput is:\n" + result.getOutput() ,
-            TaskOutcome.UP_TO_DATE,
-            task.getOutcome()
-        );
     }
 
     protected void assertBuildFileExists(BuildResult result, String projectName, String path) {
@@ -139,4 +149,16 @@ protected String getLocalTestRepoPath() {
             return file.getAbsolutePath();
         }
     }
+
+    public void assertOutputOnlyOnce(String output, String... text) {
+        for (String each : text) {
+            int i = output.indexOf(each);
+            if (i == -1 ) {
+                fail("Expected `" + text + "` to appear at most once, but it didn't at all.\n\nOutout is:\n"+ output);
+            }
+            if(output.indexOf(each) !=  output.lastIndexOf(each)) {
+                fail("Expected `" + text + "` to appear at most once, but it did multiple times.\n\nOutout is:\n"+ output);
+            }
+        }
+    }
 }
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
index f1461dbbd3d97..c6e3b2ca370ce 100644
--- a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
@@ -21,124 +21,130 @@
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
 import org.gradle.testkit.runner.GradleRunner;
-import org.gradle.testkit.runner.TaskOutcome;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import java.util.Arrays;
 
 public class TestClustersPluginIT extends GradleIntegrationTestCase {
 
     public void testListClusters() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("listTestClusters", "-s")
-            .withPluginClasspath()
-            .build();
+        BuildResult result = getTestClustersRunner("listTestClusters").build();
 
-        assertEquals(TaskOutcome.SUCCESS, result.task(":listTestClusters").getOutcome());
+        assertTaskSuccessful(result, ":listTestClusters");
         assertOutputContains(
             result.getOutput(),
-                "   * myTestCluster:"
+            "   * myTestCluster:"
         );
-
     }
 
     public void testUseClusterByOne() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("user1", "-s")
-            .withPluginClasspath()
-            .build();
-
-        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
-        assertOutputContains(
-            result.getOutput(),
-                "Starting cluster: myTestCluster",
-                "Stopping myTestCluster, number of claims is 0"
-        );
+        BuildResult result = getTestClustersRunner("user1").build();
+        assertTaskSuccessful(result, ":user1");
+        assertStartedAndStoppedOnce(result);
     }
 
     public void testUseClusterByOneWithDryRun() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("user1", "-s", "--dry-run")
-            .withPluginClasspath()
-            .build();
-
+        BuildResult result = getTestClustersRunner("--dry-run", "user1").build();
         assertNull(result.task(":user1"));
-        assertOutputDoesNotContain(
-            result.getOutput(),
-            "Starting cluster: myTestCluster",
-            "Stopping myTestCluster, number of claims is 0"
-        );
+        assertNotStarted(result);
     }
 
     public void testUseClusterByTwo() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("user1", "user2", "-s")
-            .withPluginClasspath()
-            .build();
+        BuildResult result = getTestClustersRunner("user1", "user2").build();
+        assertTaskSuccessful(result, ":user1", ":user2");
+        assertStartedAndStoppedOnce(result);
+    }
 
-        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
-        assertEquals(TaskOutcome.SUCCESS, result.task(":user2").getOutcome());
+    public void testUseClusterByUpToDateTask() {
+        BuildResult result = getTestClustersRunner("upToDate1", "upToDate2").build();
+        assertTaskUpToDate(result, ":upToDate1", ":upToDate2");
+        assertNotStarted(result);
+    }
+
+    public void testUseClusterBySkippedTask() {
+        BuildResult result = getTestClustersRunner("skipped1", "skipped2").build();
+        assertTaskSkipped(result, ":skipped1", ":skipped2");
+        assertNotStarted(result);
+    }
+
+    public void testUseClusterBySkippedAndWorkingTask() {
+        BuildResult result = getTestClustersRunner("skipped1", "user1").build();
+        assertTaskSkipped(result, ":skipped1");
+        assertTaskSuccessful(result, ":user1");
         assertOutputContains(
             result.getOutput(),
-            "Starting cluster: myTestCluster",
-            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
-            "Stopping myTestCluster, number of claims is 0"
+            "> Task :user1",
+            "Starting `ElasticsearchNode{name='myTestCluster'}`",
+            "Stopping `ElasticsearchNode{name='myTestCluster'}`"
         );
     }
 
-    public void testUseClusterByUpToDateTask() {
+    public void testMultiProject() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("upToDate1", "upToDate2", "-s")
+            .withProjectDir(getProjectDir("testclusters_multiproject"))
+            .withArguments("user1", "user2", "-s", "-i", "--parallel")
             .withPluginClasspath()
             .build();
+        assertTaskSuccessful(result, ":user1", ":user2");
+
+        assertStartedAndStoppedOnce(result);
+    }
 
-        assertEquals(TaskOutcome.UP_TO_DATE, result.task(":upToDate1").getOutcome());
-        assertEquals(TaskOutcome.UP_TO_DATE, result.task(":upToDate2").getOutcome());
+    public void testUseClusterByFailingOne() {
+        BuildResult result = getTestClustersRunner("itAlwaysFails").buildAndFail();
+        assertTaskFailed(result, ":itAlwaysFails");
+        assertStartedAndStoppedOnce(result);
         assertOutputContains(
             result.getOutput(),
-            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
-            "cluster was not running: myTestCluster"
+            "Stopping `ElasticsearchNode{name='myTestCluster'}`, tailLogs: true",
+            "Execution failed for task ':itAlwaysFails'."
         );
-        assertOutputDoesNotContain(result.getOutput(), "Starting cluster: myTestCluster");
     }
 
-    public void testUseClusterBySkippedTask() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("skipped1", "skipped2", "-s")
-            .withPluginClasspath()
-            .build();
+    public void testUseClusterByFailingDependency() {
+        BuildResult result = getTestClustersRunner("dependsOnFailed").buildAndFail();
+        assertTaskFailed(result, ":itAlwaysFails");
+        assertNull(result.task(":dependsOnFailed"));
+        assertStartedAndStoppedOnce(result);
+        assertOutputContains(
+            result.getOutput(),
+            "Stopping `ElasticsearchNode{name='myTestCluster'}`, tailLogs: true",
+            "Execution failed for task ':itAlwaysFails'."
+        );
+    }
 
-        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped1").getOutcome());
-        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped2").getOutcome());
+    public void testConfigurationLocked() {
+        BuildResult result = getTestClustersRunner("illegalConfigAlter").buildAndFail();
+        assertTaskFailed(result, ":illegalConfigAlter");
         assertOutputContains(
             result.getOutput(),
-            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
-            "cluster was not running: myTestCluster"
+            "Configuration can not be altered, already locked"
         );
-        assertOutputDoesNotContain(result.getOutput(), "Starting cluster: myTestCluster");
     }
 
-    public void tetUseClusterBySkippedAndWorkingTask() {
-        BuildResult result = GradleRunner.create()
+    private void assertNotStarted(BuildResult result) {
+        assertOutputDoesNotContain(
+            result.getOutput(),
+            "Starting ",
+            "Stopping "
+        );
+    }
+
+    private GradleRunner getTestClustersRunner(String... tasks) {
+        String[] arguments = Arrays.copyOf(tasks, tasks.length + 2);
+        arguments[tasks.length] = "-s";
+        arguments[tasks.length + 1] = "-i";
+        return GradleRunner.create()
             .withProjectDir(getProjectDir("testclusters"))
-            .withArguments("skipped1", "user1", "-s")
-            .withPluginClasspath()
-            .build();
+            .withArguments(arguments)
+            .withPluginClasspath();
+    }
 
-        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped1").getOutcome());
-        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
-        assertOutputContains(
+
+    private void assertStartedAndStoppedOnce(BuildResult result) {
+        assertOutputOnlyOnce(
             result.getOutput(),
-            "> Task :user1",
-            "Starting cluster: myTestCluster",
-            "Stopping myTestCluster, number of claims is 0"
+            "Starting `ElasticsearchNode{name='myTestCluster'}`",
+            "Stopping `ElasticsearchNode{name='myTestCluster'}`"
         );
     }
-
 }
diff --git a/buildSrc/src/testKit/testclusters/build.gradle b/buildSrc/src/testKit/testclusters/build.gradle
index 470111f056ef9..bd1cfc143f42d 100644
--- a/buildSrc/src/testKit/testclusters/build.gradle
+++ b/buildSrc/src/testKit/testclusters/build.gradle
@@ -39,3 +39,23 @@ task skipped2 {
     enabled = false
     useCluster testClusters.myTestCluster
 }
+
+task itAlwaysFails {
+    doLast {
+        throw new GradleException("Task 1 failed!")
+    }
+    useCluster testClusters.myTestCluster
+}
+
+task dependsOnFailed {
+    dependsOn itAlwaysFails
+    useCluster testClusters.myTestCluster
+}
+
+task illegalConfigAlter {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "Going to alter configuration after use"
+        testClusters.myTestCluster.distribution = 'ZIP_OSS'
+    }
+}
diff --git a/buildSrc/src/testKit/testclusters_multiproject/alpha/build.gradle b/buildSrc/src/testKit/testclusters_multiproject/alpha/build.gradle
new file mode 100644
index 0000000000000..d9f18afd68b90
--- /dev/null
+++ b/buildSrc/src/testKit/testclusters_multiproject/alpha/build.gradle
@@ -0,0 +1,18 @@
+plugins {
+    id 'elasticsearch.testclusters'
+}
+testClusters {
+    myTestCluster
+}
+task user1 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
+task user2 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
diff --git a/buildSrc/src/testKit/testclusters_multiproject/bravo/build.gradle b/buildSrc/src/testKit/testclusters_multiproject/bravo/build.gradle
new file mode 100644
index 0000000000000..2e1461f0b0f28
--- /dev/null
+++ b/buildSrc/src/testKit/testclusters_multiproject/bravo/build.gradle
@@ -0,0 +1,21 @@
+plugins {
+    id 'elasticsearch.testclusters'
+}
+
+testClusters {
+    myTestCluster
+}
+
+task user1 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
+
+task user2 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
diff --git a/buildSrc/src/testKit/testclusters_multiproject/build.gradle b/buildSrc/src/testKit/testclusters_multiproject/build.gradle
new file mode 100644
index 0000000000000..3527d1821d212
--- /dev/null
+++ b/buildSrc/src/testKit/testclusters_multiproject/build.gradle
@@ -0,0 +1,21 @@
+plugins {
+    id 'elasticsearch.testclusters'
+}
+
+testClusters {
+    myTestCluster
+}
+
+task user1 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
+
+task user2 {
+    useCluster testClusters.myTestCluster
+    doFirst {
+        println "$path"
+    }
+}
\ No newline at end of file
diff --git a/buildSrc/src/testKit/testclusters_multiproject/charlie/build.gradle b/buildSrc/src/testKit/testclusters_multiproject/charlie/build.gradle
new file mode 100644
index 0000000000000..f63a77aaea42d
--- /dev/null
+++ b/buildSrc/src/testKit/testclusters_multiproject/charlie/build.gradle
@@ -0,0 +1,5 @@
+task hello() {
+    doLast {
+        println "This task does not use the testclusters plugin. So it will have no extension."
+    }
+}
\ No newline at end of file
diff --git a/buildSrc/src/testKit/testclusters_multiproject/settings.gradle b/buildSrc/src/testKit/testclusters_multiproject/settings.gradle
new file mode 100644
index 0000000000000..aa91948920148
--- /dev/null
+++ b/buildSrc/src/testKit/testclusters_multiproject/settings.gradle
@@ -0,0 +1,3 @@
+include ':alpha'
+include ':bravo'
+include ':charlie'
\ No newline at end of file
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d8cd559cc2def..14eec1efe1c0b 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 6.6.0
-lucene            = 7.5.0
+lucene            = 7.6.0-snapshot-f9598f335b
 
 # optional dependencies
 spatial4j         = 0.7
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/GraphClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/GraphClient.java
index 3d5365fedde5a..70912b094d023 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/GraphClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/GraphClient.java
@@ -44,7 +44,7 @@ public class GraphClient {
     public final GraphExploreResponse explore(GraphExploreRequest graphExploreRequest,
                                                              RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(graphExploreRequest, GraphRequestConverters::explore,
-                options, GraphExploreResponse::fromXContext, emptySet());
+                options, GraphExploreResponse::fromXContent, emptySet());
     }
 
     /**
@@ -57,7 +57,7 @@ public final void exploreAsync(GraphExploreRequest graphExploreRequest,
                                            RequestOptions options,
                                            ActionListener<GraphExploreResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(graphExploreRequest, GraphRequestConverters::explore,
-            options, GraphExploreResponse::fromXContext, listener, emptySet());
+            options, GraphExploreResponse::fromXContent, listener, emptySet());
     }    
     
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java
new file mode 100644
index 0000000000000..88456f8dcc095
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleClient.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
+import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.StartILMRequest;
+import org.elasticsearch.client.indexlifecycle.StopILMRequest;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptySet;
+
+public class IndexLifecycleClient {
+    private final RestHighLevelClient restHighLevelClient;
+
+    IndexLifecycleClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Retrieve one or more lifecycle policy definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetLifecyclePolicyResponse getLifecyclePolicy(GetLifecyclePolicyRequest request,
+                                                         RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::getLifecyclePolicy, options,
+            GetLifecyclePolicyResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously retrieve one or more lifecycle policy definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getLifecyclePolicyAsync(GetLifecyclePolicyRequest request, RequestOptions options,
+                                        ActionListener<GetLifecyclePolicyResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::getLifecyclePolicy, options,
+            GetLifecyclePolicyResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Create or modify a lifecycle definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse putLifecyclePolicy(PutLifecyclePolicyRequest request,
+                                                   RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::putLifecyclePolicy, options,
+            AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously create or modify a lifecycle definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void putLifecyclePolicyAsync(PutLifecyclePolicyRequest request, RequestOptions options,
+                                        ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::putLifecyclePolicy, options,
+            AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Delete a lifecycle definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse deleteLifecyclePolicy(DeleteLifecyclePolicyRequest request,
+                                                      RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::deleteLifecyclePolicy, options,
+            AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously delete a lifecycle definition
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteLifecyclePolicyAsync(DeleteLifecyclePolicyRequest request, RequestOptions options,
+                                           ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::deleteLifecyclePolicy, options,
+            AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Remove the index lifecycle policy for an index
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public RemoveIndexLifecyclePolicyResponse removeIndexLifecyclePolicy(RemoveIndexLifecyclePolicyRequest request,
+            RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::removeIndexLifecyclePolicy,
+            options, RemoveIndexLifecyclePolicyResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously remove the index lifecycle policy for an index
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void removeIndexLifecyclePolicyAsync(RemoveIndexLifecyclePolicyRequest request, RequestOptions options,
+            ActionListener<RemoveIndexLifecyclePolicyResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::removeIndexLifecyclePolicy, options,
+                RemoveIndexLifecyclePolicyResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Start the Index Lifecycle Management feature.
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse startILM(StartILMRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::startILM, options,
+                AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously start the Index Lifecycle Management feature.
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void startILMAsync(StartILMRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::startILM, options,
+                AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Stop the Index Lifecycle Management feature.
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse stopILM(StopILMRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::stopILM, options,
+                AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Get the status of index lifecycle management
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public LifecycleManagementStatusResponse lifecycleManagementStatus(LifecycleManagementStatusRequest request, RequestOptions options)
+        throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::lifecycleManagementStatus,
+            options, LifecycleManagementStatusResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get the status of index lifecycle management
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     *
+     * @param request  the request
+     * @param options  the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void lifecycleManagementStatusAsync(LifecycleManagementStatusRequest request, RequestOptions options,
+                                               ActionListener<LifecycleManagementStatusResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::lifecycleManagementStatus, options,
+            LifecycleManagementStatusResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Asynchronously stop the Index Lifecycle Management feature.
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void stopILMAsync(StopILMRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::stopILM, options,
+                AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Explain the lifecycle state for an index
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ExplainLifecycleResponse explainLifecycle(ExplainLifecycleRequest request,RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::explainLifecycle, options,
+            ExplainLifecycleResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously explain the lifecycle state for an index
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void explainLifecycleAsync(ExplainLifecycleRequest request, RequestOptions options,
+                                      ActionListener<ExplainLifecycleResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::explainLifecycle, options,
+                ExplainLifecycleResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Retry lifecycle step for given indices
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public AcknowledgedResponse retryLifecycleStep(RetryLifecyclePolicyRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndexLifecycleRequestConverters::retryLifecycle, options,
+            AcknowledgedResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously retry the lifecycle step for given indices
+     * See <a href="https://fix-me-when-we-have-docs.com">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void retryLifecycleStepAsync(RetryLifecyclePolicyRequest request, RequestOptions options,
+        ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndexLifecycleRequestConverters::retryLifecycle, options,
+            AcknowledgedResponse::fromXContent, listener, emptySet());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java
new file mode 100644
index 0000000000000..0ca4f22edf282
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
+import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.StartILMRequest;
+import org.elasticsearch.client.indexlifecycle.StopILMRequest;
+import org.elasticsearch.common.Strings;
+
+import java.io.IOException;
+
+final class IndexLifecycleRequestConverters {
+
+    private IndexLifecycleRequestConverters() {}
+
+    static Request getLifecyclePolicy(GetLifecyclePolicyRequest getLifecyclePolicyRequest) {
+        String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_ilm/policy")
+                .addCommaSeparatedPathParts(getLifecyclePolicyRequest.getPolicyNames()).build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(getLifecyclePolicyRequest.masterNodeTimeout());
+        params.withTimeout(getLifecyclePolicyRequest.timeout());
+        return request;
+    }
+
+    static Request putLifecyclePolicy(PutLifecyclePolicyRequest putLifecycleRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_ilm/policy")
+            .addPathPartAsIs(putLifecycleRequest.getName())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(putLifecycleRequest.masterNodeTimeout());
+        params.withTimeout(putLifecycleRequest.timeout());
+        request.setEntity(RequestConverters.createEntity(putLifecycleRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request deleteLifecyclePolicy(DeleteLifecyclePolicyRequest deleteLifecyclePolicyRequest) {
+        Request request = new Request(HttpDelete.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_ilm/policy")
+                .addPathPartAsIs(deleteLifecyclePolicyRequest.getLifecyclePolicy())
+                .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(deleteLifecyclePolicyRequest.masterNodeTimeout());
+        params.withTimeout(deleteLifecyclePolicyRequest.timeout());
+        return request;
+    }
+
+    static Request removeIndexLifecyclePolicy(RemoveIndexLifecyclePolicyRequest removePolicyRequest) {
+        String[] indices = removePolicyRequest.indices() == null ?
+                Strings.EMPTY_ARRAY : removePolicyRequest.indices().toArray(new String[] {});
+        Request request = new Request(HttpPost.METHOD_NAME,
+                new RequestConverters.EndpointBuilder()
+                        .addCommaSeparatedPathParts(indices)
+                        .addPathPartAsIs("_ilm", "remove")
+                        .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withIndicesOptions(removePolicyRequest.indicesOptions());
+        params.withMasterTimeout(removePolicyRequest.masterNodeTimeout());
+        return request;
+    }
+
+    static Request startILM(StartILMRequest startILMRequest) {
+        Request request = new Request(HttpPost.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_ilm")
+                .addPathPartAsIs("start")
+            .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(startILMRequest.masterNodeTimeout());
+        params.withTimeout(startILMRequest.timeout());
+        return request;
+    }
+
+    static Request stopILM(StopILMRequest stopILMRequest) {
+        Request request = new Request(HttpPost.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_ilm")
+                .addPathPartAsIs("stop")
+            .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(stopILMRequest.masterNodeTimeout());
+        params.withTimeout(stopILMRequest.timeout());
+        return request;
+    }
+
+    static Request lifecycleManagementStatus(LifecycleManagementStatusRequest lifecycleManagementStatusRequest){
+        Request request = new Request(HttpGet.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addPathPartAsIs("_ilm")
+                .addPathPartAsIs("status")
+            .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(lifecycleManagementStatusRequest.masterNodeTimeout());
+        params.withTimeout(lifecycleManagementStatusRequest.timeout());
+        return request;
+    }
+
+    static Request explainLifecycle(ExplainLifecycleRequest explainLifecycleRequest) {
+        String[] indices = explainLifecycleRequest.indices() == null ? Strings.EMPTY_ARRAY : explainLifecycleRequest.indices();
+        Request request = new Request(HttpGet.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addCommaSeparatedPathParts(indices)
+                .addPathPartAsIs("_ilm")
+                .addPathPartAsIs("explain")
+            .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withIndicesOptions(explainLifecycleRequest.indicesOptions());
+        params.withMasterTimeout(explainLifecycleRequest.masterNodeTimeout());
+        return request;
+    }
+
+    static Request retryLifecycle(RetryLifecyclePolicyRequest retryLifecyclePolicyRequest) {
+        Request request = new Request(HttpPost.METHOD_NAME,
+            new RequestConverters.EndpointBuilder()
+                .addCommaSeparatedPathParts(retryLifecyclePolicyRequest.getIndices())
+                .addPathPartAsIs("_ilm")
+                .addPathPartAsIs("retry")
+                .build());
+        RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withMasterTimeout(retryLifecyclePolicyRequest.masterNodeTimeout());
+        params.withTimeout(retryLifecyclePolicyRequest.timeout());
+        return request;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
index 69b1c4a91e087..be513b0985c55 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.client.ml.PreviewDatafeedRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
+import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
@@ -463,4 +464,16 @@ static Request deleteCalendar(DeleteCalendarRequest deleteCalendarRequest) {
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
         return request;
     }
+
+    static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("filters")
+            .addPathPart(putFilterRequest.getMlFilter().getId())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putFilterRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
index 0b7647b7d579f..3f7a938f9ce1a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -61,6 +61,8 @@
 import org.elasticsearch.client.ml.PutCalendarResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
+import org.elasticsearch.client.ml.PutFilterRequest;
+import org.elasticsearch.client.ml.PutFilterResponse;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
@@ -1166,4 +1168,43 @@ public void deleteCalendarAsync(DeleteCalendarRequest request, RequestOptions op
                 listener,
                 Collections.emptySet());
     }
+
+    /**
+     * Creates a new Machine Learning Filter
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-filter.html">ML PUT Filter documentation</a>
+     *
+     * @param request The PutFilterRequest containing the {@link org.elasticsearch.client.ml.job.config.MlFilter} settings
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return PutFilterResponse with enclosed {@link org.elasticsearch.client.ml.job.config.MlFilter} object
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public PutFilterResponse putFilter(PutFilterRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::putFilter,
+            options,
+            PutFilterResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Creates a new Machine Learning Filter asynchronously and notifies listener on completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-filter.html">ML PUT Filter documentation</a>
+     *
+     * @param request  The request containing the {@link org.elasticsearch.client.ml.job.config.MlFilter} settings
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void putFilterAsync(PutFilterRequest request, RequestOptions options, ActionListener<PutFilterResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::putFilter,
+            options,
+            PutFilterResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java
new file mode 100644
index 0000000000000..24a04bd2da8d0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+
+/**
+ * Base class for responses that are node responses. These responses always contain the cluster
+ * name and the {@link NodesResponseHeader}.
+ */
+public abstract class NodesResponse {
+
+    private final NodesResponseHeader header;
+    private final String clusterName;
+
+    protected NodesResponse(NodesResponseHeader header, String clusterName) {
+        this.header = header;
+        this.clusterName = clusterName;
+    }
+
+    /**
+     * Get the cluster name associated with all of the nodes.
+     *
+     * @return Never {@code null}.
+     */
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    /**
+     * Gets information about the number of total, successful and failed nodes the request was run on.
+     * Also includes exceptions if relevant.
+     */
+    public NodesResponseHeader getHeader() {
+        return header;
+    }
+
+    public static <T extends NodesResponse> void declareCommonNodesResponseParsing(ConstructingObjectParser<T, Void> parser) {
+        parser.declareObject(ConstructingObjectParser.constructorArg(), NodesResponseHeader::fromXContent, new ParseField("_nodes"));
+        parser.declareString(ConstructingObjectParser.constructorArg(), new ParseField("cluster_name"));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index aae2cfccf521a..ca5cffa31000c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.client.security.RefreshPolicy;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Nullable;
@@ -84,6 +85,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
+import java.util.List;
 import java.util.Locale;
 import java.util.StringJoiner;
 
@@ -447,6 +449,16 @@ static Request multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplat
         return request;
     }
 
+    static Request count(CountRequest countRequest) throws IOException {
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint(countRequest.indices(), countRequest.types(), "_count"));
+        Params params = new Params(request);
+        params.withRouting(countRequest.routing());
+        params.withPreference(countRequest.preference());
+        params.withIndicesOptions(countRequest.indicesOptions());
+        request.setEntity(createEntity(countRequest.source(), REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request explain(ExplainRequest explainRequest) throws IOException {
         Request request = new Request(HttpGet.METHOD_NAME,
             endpoint(explainRequest.index(), explainRequest.type(), explainRequest.id(), "_explain"));
@@ -1022,7 +1034,12 @@ EndpointBuilder addCommaSeparatedPathParts(String[] parts) {
             return this;
         }
 
-        EndpointBuilder addPathPartAsIs(String... parts) {
+        EndpointBuilder addCommaSeparatedPathParts(List<String> parts) {
+            addPathPart(String.join(",", parts));
+            return this;
+        }
+
+        EndpointBuilder addPathPartAsIs(String ... parts) {
             for (String part : parts) {
                 if (Strings.hasLength(part)) {
                     joiner.add(part);
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index c89bb2bfd787a..6bdfaed1f0855 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -57,6 +57,8 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
 import org.elasticsearch.client.core.TermVectorsResponse;
 import org.elasticsearch.client.core.TermVectorsRequest;
 import org.elasticsearch.common.CheckedConsumer;
@@ -157,6 +159,8 @@
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.ParsedMedianAbsoluteDeviation;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
@@ -217,6 +221,7 @@ public class RestHighLevelClient implements Closeable {
     private final WatcherClient watcherClient = new WatcherClient(this);
     private final GraphClient graphClient = new GraphClient(this);
     private final LicenseClient licenseClient = new LicenseClient(this);
+    private final IndexLifecycleClient indexLifecycleClient = new IndexLifecycleClient(this);
     private final MigrationClient migrationClient = new MigrationClient(this);
     private final MachineLearningClient machineLearningClient = new MachineLearningClient(this);
     private final SecurityClient securityClient = new SecurityClient(this);
@@ -366,6 +371,16 @@ public final XPackClient xpack() {
      */
     public LicenseClient license() { return licenseClient; }
 
+    /**
+     * Provides methods for accessing the Elastic Licensed Index Lifecycle APIs that are shipped with the default distribution of
+     * Elasticsearch. All of these APIs will 404 if run against the OSS distribution of Elasticsearch.
+     * <p>
+     * See the <a href="http://FILL-ME-IN-WE-HAVE-NO-DOCS-YET.com"> X-Pack APIs on elastic.co</a> for more information.
+     */
+    public IndexLifecycleClient indexLifecycle() {
+        return indexLifecycleClient;
+    }
+
     /**
      * Provides methods for accessing the Elastic Licensed Licensing APIs that
      * are shipped with the default distribution of Elasticsearch. All of
@@ -910,6 +925,31 @@ public final void indexAsync(IndexRequest indexRequest, ActionListener<IndexResp
                 emptySet(), headers);
     }
 
+    /**
+     * Executes a count request using the Count API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html">Count API on elastic.co</a>
+     * @param countRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public final CountResponse count(CountRequest countRequest, RequestOptions options) throws IOException {
+        return performRequestAndParseEntity(countRequest, RequestConverters::count, options, CountResponse::fromXContent,
+        emptySet());
+    }
+
+    /**
+     * Asynchronously executes a count request using the Count API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html">Count API on elastic.co</a>
+     * @param countRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void countAsync(CountRequest countRequest, RequestOptions options, ActionListener<CountResponse> listener) {
+        performRequestAsyncAndParseEntity(countRequest, RequestConverters::count,  options,CountResponse::fromXContent,
+            listener, emptySet());
+    }
+
     /**
      * Updates a document using the Update API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html">Update API on elastic.co</a>
@@ -1886,6 +1926,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
         map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
         map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
+        map.put(MedianAbsoluteDeviationAggregationBuilder.NAME, (p, c) -> ParsedMedianAbsoluteDeviation.fromXContent(p, (String) c));
         map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
         map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
         map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java
index cb33384c12b48..62c7d1e9ab92c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
+import org.elasticsearch.client.rollup.DeleteRollupJobResponse;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsResponse;
 import org.elasticsearch.client.rollup.GetRollupJobRequest;
@@ -114,6 +116,40 @@ public void startRollupJobAsync(StartRollupJobRequest request, RequestOptions op
                 listener, Collections.emptySet());
     }
 
+    /**
+     * Delete a rollup job from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-delete-job.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DeleteRollupJobResponse deleteRollupJob(DeleteRollupJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            RollupRequestConverters::deleteJob,
+            options,
+            DeleteRollupJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+    /**
+     * Asynchronously delete a rollup job from the cluster
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-delete-job.html">
+     *     The docs</a> for details.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteRollupJobAsync(DeleteRollupJobRequest request,
+                                     RequestOptions options,
+                                     ActionListener<DeleteRollupJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            RollupRequestConverters::deleteJob,
+            options,
+            DeleteRollupJobResponse::fromXContent,
+            listener, Collections.emptySet());
+    }
+
     /**
      * Get a rollup job from the cluster.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-put-job.html">
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
index 3f780eca059ef..8d1f07dae4fdf 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
@@ -18,10 +18,12 @@
  */
 package org.elasticsearch.client;
 
+import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
+import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupJobRequest;
 import org.elasticsearch.client.rollup.PutRollupJobRequest;
 import org.elasticsearch.client.rollup.StartRollupJobRequest;
@@ -74,4 +76,16 @@ static Request getRollupCaps(final GetRollupCapsRequest getRollupCapsRequest) th
         request.setEntity(createEntity(getRollupCapsRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
+
+    static Request deleteJob(final DeleteRollupJobRequest deleteRollupJobRequest) throws IOException {
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("rollup")
+            .addPathPartAsIs("job")
+            .addPathPart(deleteRollupJobRequest.getId())
+            .build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(deleteRollupJobRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
index e604814a3bce5..75103b7ad5602 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
@@ -20,7 +20,11 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.security.AuthenticateRequest;
+import org.elasticsearch.client.security.AuthenticateResponse;
 import org.elasticsearch.client.security.ChangePasswordRequest;
+import org.elasticsearch.client.security.ClearRealmCacheRequest;
+import org.elasticsearch.client.security.ClearRealmCacheResponse;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
@@ -211,13 +215,69 @@ public void disableUserAsync(DisableUserRequest request, RequestOptions options,
     }
 
     /**
-     * Clears the native roles cache for a set of roles.
+     * Authenticate the current user and return all the information about the authenticated user.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html">
+     * the docs</a> for more.
+     *
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized 
+     * @return the responsee from the authenticate user call
+     */
+    public AuthenticateResponse authenticate(RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(AuthenticateRequest.INSTANCE, AuthenticateRequest::getRequest, options,
+                AuthenticateResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Authenticate the current user asynchronously and return all the information about the authenticated user.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html">
+     * the docs</a> for more.
+     *
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized 
+     * @param listener the listener to be notified upon request completion 
+     */
+    public void authenticateAsync(RequestOptions options, ActionListener<AuthenticateResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(AuthenticateRequest.INSTANCE, AuthenticateRequest::getRequest, options,
+                AuthenticateResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Clears the cache in one or more realms.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-cache.html">
+     * the docs</a> for more.
+     *
+     * @param request the request with the realm names and usernames to clear the cache for
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the clear realm cache call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ClearRealmCacheResponse clearRealmCache(ClearRealmCacheRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::clearRealmCache, options,
+            ClearRealmCacheResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Clears the cache in one or more realms asynchronously.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-cache.html">
+     * the docs</a> for more.
+     *
+     * @param request  the request with the realm names and usernames to clear the cache for
+     * @param options  the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void clearRealmCacheAsync(ClearRealmCacheRequest request, RequestOptions options,
+                                     ActionListener<ClearRealmCacheResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::clearRealmCache, options,
+            ClearRealmCacheResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Clears the roles cache for a set of roles.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-role-cache.html">
      * the docs</a> for more.
      *
      * @param request the request with the roles for which the cache should be cleared.
      * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response from the enable user call
+     * @return the response from the clear roles cache call
      * @throws IOException in case there is a problem sending the request or parsing back the response
      */
     public ClearRolesCacheResponse clearRolesCache(ClearRolesCacheRequest request, RequestOptions options) throws IOException {
@@ -226,7 +286,7 @@ public ClearRolesCacheResponse clearRolesCache(ClearRolesCacheRequest request, R
     }
 
     /**
-     * Clears the native roles cache for a set of roles asynchronously.
+     * Clears the roles cache for a set of roles asynchronously.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-role-cache.html">
      * the docs</a> for more.
      *
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
index 2feb15704333e..fe0a9833cb672 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
@@ -23,6 +23,7 @@
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.client.security.ClearRealmCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
@@ -111,6 +112,23 @@ private static Request setUserEnabled(SetUserEnabledRequest setUserEnabledReques
         return request;
     }
 
+    static Request clearRealmCache(ClearRealmCacheRequest clearRealmCacheRequest) {
+        RequestConverters.EndpointBuilder builder = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_xpack/security/realm");
+        if (clearRealmCacheRequest.getRealms().isEmpty() == false) {
+            builder.addCommaSeparatedPathParts(clearRealmCacheRequest.getRealms().toArray(Strings.EMPTY_ARRAY));
+        } else {
+            builder.addPathPart("_all");
+        }
+        final String endpoint = builder.addPathPartAsIs("_clear_cache").build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        if (clearRealmCacheRequest.getUsernames().isEmpty() == false) {
+            RequestConverters.Params params = new RequestConverters.Params(request);
+            params.putParam("usernames", Strings.collectionToCommaDelimitedString(clearRealmCacheRequest.getUsernames()));
+        }
+        return request;
+    }
+
     static Request clearRolesCache(ClearRolesCacheRequest disableCacheRequest) {
         String endpoint = new RequestConverters.EndpointBuilder()
             .addPathPartAsIs("_xpack/security/role")
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java
index c26a7ba48ca17..60ecea39ae093 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java
@@ -28,7 +28,7 @@
  * Please note, any requests that use a ackTimeout should set timeout as they
  * represent the same backing field on the server.
  */
-public class TimedRequest implements Validatable {
+public abstract class TimedRequest implements Validatable {
 
     public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30);
     public static final TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index a20dfd1ba328a..1e8a5328355a0 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -94,4 +94,5 @@ public void usageAsync(XPackUsageRequest request, RequestOptions options, Action
         restHighLevelClient.performRequestAsyncAndParseEntity(request, XPackRequestConverters::usage, options,
             XPackUsageResponse::fromXContent, listener, emptySet());
     }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java
new file mode 100644
index 0000000000000..6d4589c7861f6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.core;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.elasticsearch.action.search.SearchRequest.DEFAULT_INDICES_OPTIONS;
+
+/**
+ * Encapsulates a request to _count API against one, several or all indices.
+ */
+public final class CountRequest extends ActionRequest implements IndicesRequest.Replaceable {
+
+    private String[] indices = Strings.EMPTY_ARRAY;
+    private String[] types = Strings.EMPTY_ARRAY;
+    private String routing;
+    private String preference;
+    private SearchSourceBuilder searchSourceBuilder;
+    private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
+
+    public CountRequest() {
+        this.searchSourceBuilder = new SearchSourceBuilder();
+    }
+
+    /**
+     * Constructs a new count request against the indices. No indices provided here means that count will execute on all indices.
+     */
+    public CountRequest(String... indices) {
+        this(indices, new SearchSourceBuilder());
+    }
+
+    /**
+     * Constructs a new search request against the provided indices with the given search source.
+     */
+    public CountRequest(String[] indices, SearchSourceBuilder searchSourceBuilder) {
+        indices(indices);
+        this.searchSourceBuilder = searchSourceBuilder;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    /**
+     * Sets the indices the count will be executed on.
+     */
+    public CountRequest indices(String... indices) {
+        Objects.requireNonNull(indices, "indices must not be null");
+        for (String index : indices) {
+            Objects.requireNonNull(index, "index must not be null");
+        }
+        this.indices = indices;
+        return this;
+    }
+
+    /**
+     * The source of the count request.
+     */
+    public CountRequest source(SearchSourceBuilder searchSourceBuilder) {
+        this.searchSourceBuilder = Objects.requireNonNull(searchSourceBuilder, "source must not be null");
+        return this;
+    }
+
+    /**
+     * The document types to execute the count against. Defaults to be executed against all types.
+     *
+     * @deprecated Types are going away, prefer filtering on a type.
+     */
+    @Deprecated
+    public CountRequest types(String... types) {
+        Objects.requireNonNull(types, "types must not be null");
+        for (String type : types) {
+            Objects.requireNonNull(type, "type must not be null");
+        }
+        this.types = types;
+        return this;
+    }
+
+    /**
+     * The routing values to control the shards that the search will be executed on.
+     */
+    public CountRequest routing(String routing) {
+        this.routing = routing;
+        return this;
+    }
+
+    /**
+     * A comma separated list of routing values to control the shards the count will be executed on.
+     */
+    public CountRequest routing(String... routings) {
+        this.routing = Strings.arrayToCommaDelimitedString(routings);
+        return this;
+    }
+
+    /**
+     * Returns the indices options used to resolve indices. They tell for instance whether a single index is accepted, whether an empty
+     * array will be converted to _all, and how wildcards will be expanded if needed.
+     *
+     * @see org.elasticsearch.action.support.IndicesOptions
+     */
+    public CountRequest indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = Objects.requireNonNull(indicesOptions, "indicesOptions must not be null");
+        return this;
+    }
+
+    /**
+     * Sets the preference to execute the count. Defaults to randomize across shards. Can be set to {@code _local} to prefer local shards
+     * or a custom value, which guarantees that the same order will be used across different requests.
+     */
+    public CountRequest preference(String preference) {
+        this.preference = preference;
+        return this;
+    }
+
+    public IndicesOptions indicesOptions() {
+        return this.indicesOptions;
+    }
+
+    public String routing() {
+        return this.routing;
+    }
+
+    public String preference() {
+        return this.preference;
+    }
+
+    public String[] indices() {
+        return Arrays.copyOf(this.indices, this.indices.length);
+    }
+
+    public Float minScore() {
+        return this.searchSourceBuilder.minScore();
+    }
+
+    public CountRequest minScore(Float minScore) {
+        this.searchSourceBuilder.minScore(minScore);
+        return this;
+    }
+
+    public int terminateAfter() {
+        return this.searchSourceBuilder.terminateAfter();
+    }
+
+    public CountRequest terminateAfter(int terminateAfter) {
+        this.searchSourceBuilder.terminateAfter(terminateAfter);
+        return this;
+    }
+
+    public String[] types() {
+        return Arrays.copyOf(this.types, this.types.length);
+    }
+
+    public SearchSourceBuilder source() {
+        return this.searchSourceBuilder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        CountRequest that = (CountRequest) o;
+        return Objects.equals(indicesOptions, that.indicesOptions) &&
+            Arrays.equals(indices, that.indices) &&
+            Arrays.equals(types, that.types) &&
+            Objects.equals(routing, that.routing) &&
+            Objects.equals(preference, that.preference);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(indicesOptions, routing, preference);
+        result = 31 * result + Arrays.hashCode(indices);
+        result = 31 * result + Arrays.hashCode(types);
+        return result;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java
new file mode 100644
index 0000000000000..f97f79127e690
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.core;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * A response to _count API request.
+ */
+public final class CountResponse extends ActionResponse {
+
+    static final ParseField COUNT = new ParseField("count");
+    static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
+    static final ParseField SHARDS = new ParseField("_shards");
+
+    private final long count;
+    private final Boolean terminatedEarly;
+    private final ShardStats shardStats;
+
+    public CountResponse(long count, Boolean terminatedEarly, ShardStats shardStats) {
+        this.count = count;
+        this.terminatedEarly = terminatedEarly;
+        this.shardStats = shardStats;
+    }
+
+    public ShardStats getShardStats() {
+        return shardStats;
+    }
+
+    /**
+     * Number of documents matching request.
+     */
+    public long getCount() {
+        return count;
+    }
+
+    /**
+     * The total number of shards the search was executed on.
+     */
+    public int getTotalShards() {
+        return shardStats.totalShards;
+    }
+
+    /**
+     * The successful number of shards the search was executed on.
+     */
+    public int getSuccessfulShards() {
+        return shardStats.successfulShards;
+    }
+
+    /**
+     * The number of shards skipped due to pre-filtering
+     */
+    public int getSkippedShards() {
+        return shardStats.skippedShards;
+    }
+
+    /**
+     * The failed number of shards the search was executed on.
+     */
+    public int getFailedShards() {
+        return shardStats.shardFailures.length;
+    }
+
+    /**
+     * The failures that occurred during the search.
+     */
+    public ShardSearchFailure[] getShardFailures() {
+        return shardStats.shardFailures;
+    }
+
+    public RestStatus status() {
+        return RestStatus.status(shardStats.successfulShards, shardStats.totalShards, shardStats.shardFailures);
+    }
+
+    public static CountResponse fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+        String currentName = parser.currentName();
+        Boolean terminatedEarly = null;
+        long count = 0;
+        ShardStats shardStats = new ShardStats(-1, -1,0, ShardSearchFailure.EMPTY_ARRAY);
+
+        for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentName = parser.currentName();
+            } else if (token.isValue()) {
+                if (COUNT.match(currentName, parser.getDeprecationHandler())) {
+                    count = parser.longValue();
+                } else if (TERMINATED_EARLY.match(currentName, parser.getDeprecationHandler())) {
+                    terminatedEarly = parser.booleanValue();
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (SHARDS.match(currentName, parser.getDeprecationHandler())) {
+                    shardStats = ShardStats.fromXContent(parser);
+                } else {
+                    parser.skipChildren();
+                }
+            }
+        }
+        return new CountResponse(count, terminatedEarly, shardStats);
+    }
+
+    @Override
+    public String toString() {
+        String s = "{" +
+            "count=" + count +
+            (isTerminatedEarly() != null ? ", terminatedEarly=" + terminatedEarly : "") +
+            ", " + shardStats +
+            '}';
+        return s;
+    }
+
+    public Boolean isTerminatedEarly() {
+        return terminatedEarly;
+    }
+
+    /**
+     * Encapsulates _shards section of count api response.
+     */
+    public static final class ShardStats {
+
+        static final ParseField FAILED = new ParseField("failed");
+        static final ParseField SKIPPED = new ParseField("skipped");
+        static final ParseField TOTAL = new ParseField("total");
+        static final ParseField SUCCESSFUL = new ParseField("successful");
+        static final ParseField FAILURES = new ParseField("failures");
+
+        private final int successfulShards;
+        private final int totalShards;
+        private final int skippedShards;
+        private final ShardSearchFailure[] shardFailures;
+
+        public ShardStats(int successfulShards, int totalShards, int skippedShards, ShardSearchFailure[] shardFailures) {
+            this.successfulShards = successfulShards;
+            this.totalShards = totalShards;
+            this.skippedShards = skippedShards;
+            this.shardFailures = Arrays.copyOf(shardFailures, shardFailures.length);
+        }
+
+        public int getSuccessfulShards() {
+            return successfulShards;
+        }
+
+        public int getTotalShards() {
+            return totalShards;
+        }
+
+        public int getSkippedShards() {
+            return skippedShards;
+        }
+
+        public ShardSearchFailure[] getShardFailures() {
+            return Arrays.copyOf(shardFailures, shardFailures.length, ShardSearchFailure[].class);
+        }
+
+        static ShardStats fromXContent(XContentParser parser) throws IOException {
+            int successfulShards = -1;
+            int totalShards = -1;
+            int skippedShards = 0; //BWC @see org.elasticsearch.action.search.SearchResponse
+            List<ShardSearchFailure> failures = new ArrayList<>();
+            XContentParser.Token token;
+            String currentName = parser.currentName();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (FAILED.match(currentName, parser.getDeprecationHandler())) {
+                        parser.intValue();
+                    } else if (SKIPPED.match(currentName, parser.getDeprecationHandler())) {
+                        skippedShards = parser.intValue();
+                    } else if (TOTAL.match(currentName, parser.getDeprecationHandler())) {
+                        totalShards = parser.intValue();
+                    } else if (SUCCESSFUL.match(currentName, parser.getDeprecationHandler())) {
+                        successfulShards = parser.intValue();
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if (FAILURES.match(currentName, parser.getDeprecationHandler())) {
+                        while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            failures.add(ShardSearchFailure.fromXContent(parser));
+                        }
+                    } else {
+                        parser.skipChildren();
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+            return new ShardStats(successfulShards, totalShards, skippedShards, failures.toArray(new ShardSearchFailure[failures.size()]));
+        }
+
+        @Override
+        public String toString() {
+            return "_shards : {" +
+                "total=" + totalShards +
+                ", successful=" + successfulShards +
+                ", skipped=" + skippedShards +
+                ", failed=" + (shardFailures != null && shardFailures.length > 0 ? shardFailures.length : 0 ) +
+                (shardFailures != null && shardFailures.length > 0 ? ", failures: " + Arrays.asList(shardFailures): "") +
+                '}';
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java
index 2b5d1c7ecf4be..dddc4bedfe466 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/GraphExploreResponse.java
@@ -47,7 +47,7 @@
  * 
  * @see GraphExploreRequest
  */
-public class GraphExploreResponse  implements ToXContentObject {
+public class GraphExploreResponse implements ToXContentObject {
 
     private long tookInMillis;
     private boolean timedOut = false;
@@ -94,14 +94,30 @@ public Collection<Connection> getConnections() {
         return connections.values();
     }
 
+    public Collection<ConnectionId> getConnectionIds() {
+        return connections.keySet();
+    }
+
+    public Connection getConnection(ConnectionId connectionId) {
+        return connections.get(connectionId);
+    }
+
     public Collection<Vertex> getVertices() {
         return vertices.values();
     }
-    
+
+    public Collection<VertexId> getVertexIds() {
+        return vertices.keySet();
+    }
+
     public Vertex getVertex(VertexId id) {
         return vertices.get(id);
     }
 
+    public boolean isReturnDetailedInfo() {
+        return returnDetailedInfo;
+    }
+
     private static final ParseField TOOK = new ParseField("took");
     private static final ParseField TIMED_OUT = new ParseField("timed_out");
     private static final ParseField VERTICES = new ParseField("vertices");
@@ -190,7 +206,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.fromXContent(p), FAILURES);
     } 
     
-    public static GraphExploreResponse fromXContext(XContentParser parser) throws IOException {
+    public static GraphExploreResponse fromXContent(XContentParser parser) throws IOException {
         return PARSER.apply(parser, null);
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/Vertex.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/Vertex.java
index 852372209da9c..54b0b5223277d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/Vertex.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/graph/Vertex.java
@@ -220,6 +220,14 @@ public VertexId(String field, String term) {
             this.term = term;
         }
 
+        public String getField() {
+            return field;
+        }
+
+        public String getTerm() {
+            return term;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o)
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java
new file mode 100644
index 0000000000000..702db15b965c7
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+public class AllocateAction implements LifecycleAction, ToXContentObject {
+
+    public static final String NAME = "allocate";
+    static final ParseField NUMBER_OF_REPLICAS_FIELD = new ParseField("number_of_replicas");
+    static final ParseField INCLUDE_FIELD = new ParseField("include");
+    static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
+    static final ParseField REQUIRE_FIELD = new ParseField("require");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+            a -> new AllocateAction((Integer) a[0], (Map<String, String>) a[1], (Map<String, String>) a[2], (Map<String, String>) a[3]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUMBER_OF_REPLICAS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), INCLUDE_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), EXCLUDE_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), REQUIRE_FIELD);
+    }
+
+    private final Integer numberOfReplicas;
+    private final Map<String, String> include;
+    private final Map<String, String> exclude;
+    private final Map<String, String> require;
+
+    public static AllocateAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public AllocateAction(Integer numberOfReplicas, Map<String, String> include, Map<String, String> exclude, Map<String, String> require) {
+        if (include == null) {
+            this.include = Collections.emptyMap();
+        } else {
+            this.include = include;
+        }
+        if (exclude == null) {
+            this.exclude = Collections.emptyMap();
+        } else {
+            this.exclude = exclude;
+        }
+        if (require == null) {
+            this.require = Collections.emptyMap();
+        } else {
+            this.require = require;
+        }
+        if (this.include.isEmpty() && this.exclude.isEmpty() && this.require.isEmpty() && numberOfReplicas == null) {
+            throw new IllegalArgumentException(
+                    "At least one of " + INCLUDE_FIELD.getPreferredName() + ", " + EXCLUDE_FIELD.getPreferredName() + " or "
+                            + REQUIRE_FIELD.getPreferredName() + "must contain attributes for action " + NAME);
+        }
+        if (numberOfReplicas != null && numberOfReplicas < 0) {
+            throw new IllegalArgumentException("[" + NUMBER_OF_REPLICAS_FIELD.getPreferredName() + "] must be >= 0");
+        }
+        this.numberOfReplicas = numberOfReplicas;
+    }
+
+    public Integer getNumberOfReplicas() {
+        return numberOfReplicas;
+    }
+
+    public Map<String, String> getInclude() {
+        return include;
+    }
+
+    public Map<String, String> getExclude() {
+        return exclude;
+    }
+
+    public Map<String, String> getRequire() {
+        return require;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        if (numberOfReplicas != null) {
+            builder.field(NUMBER_OF_REPLICAS_FIELD.getPreferredName(), numberOfReplicas);
+        }
+        builder.field(INCLUDE_FIELD.getPreferredName(), include);
+        builder.field(EXCLUDE_FIELD.getPreferredName(), exclude);
+        builder.field(REQUIRE_FIELD.getPreferredName(), require);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(numberOfReplicas, include, exclude, require);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        AllocateAction other = (AllocateAction) obj;
+        return Objects.equals(numberOfReplicas, other.numberOfReplicas) &&
+            Objects.equals(include, other.include) &&
+            Objects.equals(exclude, other.exclude) &&
+            Objects.equals(require, other.require);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java
new file mode 100644
index 0000000000000..299b0ac582771
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class DeleteAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "delete";
+
+    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, DeleteAction::new);
+
+    public static DeleteAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public DeleteAction() {
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public int hashCode() {
+        return 1;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequest.java
new file mode 100644
index 0000000000000..fc029f37ac928
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+import org.elasticsearch.common.Strings;
+
+import java.util.Objects;
+
+public class DeleteLifecyclePolicyRequest extends TimedRequest {
+
+    private final String lifecyclePolicy;
+
+    public DeleteLifecyclePolicyRequest(String lifecyclePolicy) {
+        if (Strings.isNullOrEmpty(lifecyclePolicy)) {
+            throw new IllegalArgumentException("lifecycle name must be present");
+        }
+        this.lifecyclePolicy = lifecyclePolicy;
+    }
+
+    public String getLifecyclePolicy() {
+        return lifecyclePolicy;
+    }
+
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        DeleteLifecyclePolicyRequest that = (DeleteLifecyclePolicyRequest) o;
+        return Objects.equals(getLifecyclePolicy(), that.getLifecyclePolicy());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getLifecyclePolicy());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequest.java
new file mode 100644
index 0000000000000..9d9e80bf1eeee
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.TimedRequest;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.Strings;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * The request object used by the Explain Lifecycle API.
+ *
+ * Multiple indices may be queried in the same request using the
+ * {@link #indices(String...)} method
+ */
+public class ExplainLifecycleRequest extends TimedRequest {
+
+    private String[] indices = Strings.EMPTY_ARRAY;
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+
+    public ExplainLifecycleRequest() {
+        super();
+    }
+
+    public ExplainLifecycleRequest indices(String... indices) {
+        this.indices = indices;
+        return this;
+    }
+
+    public String[] indices() {
+        return indices;
+    }
+
+    public ExplainLifecycleRequest indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = indicesOptions;
+        return this;
+    }
+
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+    
+    @Override
+    public Optional<ValidationException> validate() {
+        return Optional.empty();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.hashCode(indices()), indicesOptions());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ExplainLifecycleRequest other = (ExplainLifecycleRequest) obj;
+        return Objects.deepEquals(indices(), other.indices()) &&
+                Objects.equals(indicesOptions(), other.indicesOptions());
+    }
+
+    @Override
+    public String toString() {
+        return "ExplainLifecycleRequest [indices()=" + Arrays.toString(indices()) + ", indicesOptions()=" + indicesOptions() + "]";
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponse.java
new file mode 100644
index 0000000000000..de2803afe5415
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponse.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * The response object returned by the Explain Lifecycle API.
+ *
+ * Since the API can be run over multiple indices the response provides a map of
+ * index to the explanation of the lifecycle status for that index.
+ */
+public class ExplainLifecycleResponse implements ToXContentObject {
+
+    private static final ParseField INDICES_FIELD = new ParseField("indices");
+
+    private Map<String, IndexLifecycleExplainResponse> indexResponses;
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ExplainLifecycleResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "explain_lifecycle_response", a -> new ExplainLifecycleResponse(((List<IndexLifecycleExplainResponse>) a[0]).stream()
+                    .collect(Collectors.toMap(IndexLifecycleExplainResponse::getIndex, Function.identity()))));
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> IndexLifecycleExplainResponse.PARSER.apply(p, c),
+                INDICES_FIELD);
+    }
+
+    public static ExplainLifecycleResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ExplainLifecycleResponse(Map<String, IndexLifecycleExplainResponse> indexResponses) {
+        this.indexResponses = indexResponses;
+    }
+
+    /**
+     * @return a map of the responses from each requested index. The maps key is
+     *         the index name and the value is the
+     *         {@link IndexLifecycleExplainResponse} describing the current
+     *         lifecycle status of that index
+     */
+    public Map<String, IndexLifecycleExplainResponse> getIndexResponses() {
+        return indexResponses;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startObject(INDICES_FIELD.getPreferredName());
+        for (IndexLifecycleExplainResponse indexResponse : indexResponses.values()) {
+            builder.field(indexResponse.getIndex(), indexResponse);
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indexResponses);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ExplainLifecycleResponse other = (ExplainLifecycleResponse) obj;
+        return Objects.equals(indexResponses, other.indexResponses);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java
new file mode 100644
index 0000000000000..eb564b7cd27b6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ForceMergeAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "forcemerge";
+    private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
+
+    private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+        false, a -> {
+        int maxNumSegments = (int) a[0];
+        return new ForceMergeAction(maxNumSegments);
+    });
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), MAX_NUM_SEGMENTS_FIELD);
+    }
+
+    private final int maxNumSegments;
+
+    public static ForceMergeAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ForceMergeAction(int maxNumSegments) {
+        if (maxNumSegments <= 0) {
+            throw new IllegalArgumentException("[" + MAX_NUM_SEGMENTS_FIELD.getPreferredName()
+                + "] must be a positive integer");
+        }
+        this.maxNumSegments = maxNumSegments;
+    }
+
+    public int getMaxNumSegments() {
+        return maxNumSegments;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(MAX_NUM_SEGMENTS_FIELD.getPreferredName(), maxNumSegments);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxNumSegments);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ForceMergeAction other = (ForceMergeAction) obj;
+        return Objects.equals(maxNumSegments, other.maxNumSegments);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequest.java
new file mode 100644
index 0000000000000..af17a3ea48cf9
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+import org.elasticsearch.common.Strings;
+
+import java.util.Arrays;
+
+public class GetLifecyclePolicyRequest extends TimedRequest {
+
+    private final String[] policyNames;
+
+    public GetLifecyclePolicyRequest(String... policyNames) {
+        if (policyNames == null) {
+            this.policyNames = Strings.EMPTY_ARRAY;
+        } else {
+            for (String name : policyNames) {
+                if (name == null) {
+                    throw new IllegalArgumentException("cannot include null policy name");
+                }
+            }
+            this.policyNames = policyNames;
+        }
+    }
+
+    public String[] getPolicyNames() {
+        return policyNames;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        GetLifecyclePolicyRequest request = (GetLifecyclePolicyRequest) o;
+        return Arrays.equals(getPolicyNames(), request.getPolicyNames());
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(getPolicyNames());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponse.java
new file mode 100644
index 0000000000000..fc007cb5aebd4
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponse.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.client.indexlifecycle;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class GetLifecyclePolicyResponse implements ToXContentObject {
+
+    private final ImmutableOpenMap<String, LifecyclePolicyMetadata> policies;
+
+    public GetLifecyclePolicyResponse(ImmutableOpenMap<String, LifecyclePolicyMetadata> policies) {
+        this.policies = policies;
+    }
+
+    public ImmutableOpenMap<String, LifecyclePolicyMetadata> getPolicies() {
+        return policies;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        for (ObjectObjectCursor<String, LifecyclePolicyMetadata> stringLifecyclePolicyObjectObjectCursor : policies) {
+            builder.field(stringLifecyclePolicyObjectObjectCursor.key, stringLifecyclePolicyObjectObjectCursor.value);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public static GetLifecyclePolicyResponse fromXContent(XContentParser parser) throws IOException {
+        ImmutableOpenMap.Builder<String, LifecyclePolicyMetadata> policies = ImmutableOpenMap.builder();
+
+        if (parser.currentToken() == null) {
+            parser.nextToken();
+        }
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        parser.nextToken();
+
+        while (!parser.isClosed()) {
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                String policyName = parser.currentName();
+                LifecyclePolicyMetadata policyDefinion = LifecyclePolicyMetadata.parse(parser, policyName);
+                policies.put(policyName, policyDefinion);
+            } else {
+                parser.nextToken();
+            }
+        }
+
+        return new GetLifecyclePolicyResponse(policies.build());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        GetLifecyclePolicyResponse that = (GetLifecyclePolicyResponse) o;
+        return Objects.equals(getPolicies(), that.getPolicies());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getPolicies());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java
new file mode 100644
index 0000000000000..58ba7e63c03a7
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class IndexLifecycleExplainResponse implements ToXContentObject {
+
+    private static final ParseField INDEX_FIELD = new ParseField("index");
+    private static final ParseField MANAGED_BY_ILM_FIELD = new ParseField("managed");
+    private static final ParseField POLICY_NAME_FIELD = new ParseField("policy");
+    private static final ParseField LIFECYCLE_DATE_MILLIS_FIELD = new ParseField("lifecycle_date_millis");
+    private static final ParseField LIFECYCLE_DATE_FIELD = new ParseField("lifecycle_date");
+    private static final ParseField PHASE_FIELD = new ParseField("phase");
+    private static final ParseField ACTION_FIELD = new ParseField("action");
+    private static final ParseField STEP_FIELD = new ParseField("step");
+    private static final ParseField FAILED_STEP_FIELD = new ParseField("failed_step");
+    private static final ParseField PHASE_TIME_MILLIS_FIELD = new ParseField("phase_time_millis");
+    private static final ParseField PHASE_TIME_FIELD = new ParseField("phase_time");
+    private static final ParseField ACTION_TIME_MILLIS_FIELD = new ParseField("action_time_millis");
+    private static final ParseField ACTION_TIME_FIELD = new ParseField("action_time");
+    private static final ParseField STEP_TIME_MILLIS_FIELD = new ParseField("step_time_millis");
+    private static final ParseField STEP_TIME_FIELD = new ParseField("step_time");
+    private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
+    private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
+
+    public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "index_lifecycle_explain_response",
+        a -> new IndexLifecycleExplainResponse(
+            (String) a[0],
+            (boolean) a[1],
+            (String) a[2],
+            (long) (a[3] == null ? -1L: a[3]),
+            (String) a[4],
+            (String) a[5],
+            (String) a[6],
+            (String) a[7],
+            (long) (a[8] == null ? -1L: a[8]),
+            (long) (a[9] == null ? -1L: a[9]),
+            (long) (a[10] == null ? -1L: a[10]),
+            (BytesReference) a[11],
+            (PhaseExecutionInfo) a[12]));
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), MANAGED_BY_ILM_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), POLICY_NAME_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LIFECYCLE_DATE_MILLIS_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PHASE_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ACTION_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), STEP_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FAILED_STEP_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PHASE_TIME_MILLIS_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ACTION_TIME_MILLIS_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), STEP_TIME_MILLIS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.copyCurrentStructure(p);
+            return BytesArray.bytes(builder);
+        }, STEP_INFO_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> PhaseExecutionInfo.parse(p, ""),
+            PHASE_EXECUTION_INFO);
+    }
+
+    private final String index;
+    private final String policyName;
+    private final String phase;
+    private final String action;
+    private final String step;
+    private final String failedStep;
+    private final long lifecycleDate;
+    private final long phaseTime;
+    private final long actionTime;
+    private final long stepTime;
+    private final boolean managedByILM;
+    private final BytesReference stepInfo;
+    private final PhaseExecutionInfo phaseExecutionInfo;
+
+    public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, long lifecycleDate,
+                                                                        String phase, String action, String step, String failedStep,
+                                                                        long phaseTime, long actionTime, long stepTime,
+                                                                        BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
+        return new IndexLifecycleExplainResponse(index, true, policyName, lifecycleDate, phase, action, step, failedStep, phaseTime,
+            actionTime, stepTime, stepInfo, phaseExecutionInfo);
+    }
+
+    public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String index) {
+        return new IndexLifecycleExplainResponse(index, false, null, -1L, null, null, null, null, -1L, -1L, -1L, null, null);
+    }
+
+    private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, long lifecycleDate,
+                                          String phase, String action, String step, String failedStep, long phaseTime, long actionTime,
+                                          long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
+        if (managedByILM) {
+            if (policyName == null) {
+                throw new IllegalArgumentException("[" + POLICY_NAME_FIELD.getPreferredName() + "] cannot be null for managed index");
+            }
+        } else {
+            if (policyName != null || lifecycleDate >= 0 || phase != null || action != null || step != null || failedStep != null
+                || phaseTime >= 0 || actionTime >= 0 || stepTime >= 0 || stepInfo != null || phaseExecutionInfo != null) {
+                throw new IllegalArgumentException(
+                    "Unmanaged index response must only contain fields: [" + MANAGED_BY_ILM_FIELD + ", " + INDEX_FIELD + "]");
+            }
+        }
+        this.index = index;
+        this.policyName = policyName;
+        this.managedByILM = managedByILM;
+        this.lifecycleDate = lifecycleDate;
+        this.phase = phase;
+        this.action = action;
+        this.step = step;
+        this.phaseTime = phaseTime;
+        this.actionTime = actionTime;
+        this.stepTime = stepTime;
+        this.failedStep = failedStep;
+        this.stepInfo = stepInfo;
+        this.phaseExecutionInfo = phaseExecutionInfo;
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public boolean managedByILM() {
+        return managedByILM;
+    }
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    public long getLifecycleDate() {
+        return lifecycleDate;
+    }
+
+    public String getPhase() {
+        return phase;
+    }
+
+    public long getPhaseTime() {
+        return phaseTime;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public long getActionTime() {
+        return actionTime;
+    }
+
+    public String getStep() {
+        return step;
+    }
+
+    public long getStepTime() {
+        return stepTime;
+    }
+
+    public String getFailedStep() {
+        return failedStep;
+    }
+
+    public BytesReference getStepInfo() {
+        return stepInfo;
+    }
+
+    public PhaseExecutionInfo getPhaseExecutionInfo() {
+        return phaseExecutionInfo;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX_FIELD.getPreferredName(), index);
+        builder.field(MANAGED_BY_ILM_FIELD.getPreferredName(), managedByILM);
+        if (managedByILM) {
+            builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
+            builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
+            builder.field(PHASE_FIELD.getPreferredName(), phase);
+            builder.timeField(PHASE_TIME_MILLIS_FIELD.getPreferredName(), PHASE_TIME_FIELD.getPreferredName(), phaseTime);
+            builder.field(ACTION_FIELD.getPreferredName(), action);
+            builder.timeField(ACTION_TIME_MILLIS_FIELD.getPreferredName(), ACTION_TIME_FIELD.getPreferredName(), actionTime);
+            builder.field(STEP_FIELD.getPreferredName(), step);
+            builder.timeField(STEP_TIME_MILLIS_FIELD.getPreferredName(), STEP_TIME_FIELD.getPreferredName(), stepTime);
+            if (Strings.hasLength(failedStep)) {
+                builder.field(FAILED_STEP_FIELD.getPreferredName(), failedStep);
+            }
+            if (stepInfo != null && stepInfo.length() > 0) {
+                builder.rawField(STEP_INFO_FIELD.getPreferredName(), stepInfo.streamInput(), XContentType.JSON);
+            }
+            if (phaseExecutionInfo != null) {
+                builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo);
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, managedByILM, policyName, lifecycleDate, phase, action, step, failedStep, phaseTime, actionTime,
+            stepTime, stepInfo, phaseExecutionInfo);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        IndexLifecycleExplainResponse other = (IndexLifecycleExplainResponse) obj;
+        return Objects.equals(index, other.index) &&
+            Objects.equals(managedByILM, other.managedByILM) &&
+            Objects.equals(policyName, other.policyName) &&
+            Objects.equals(lifecycleDate, other.lifecycleDate) &&
+            Objects.equals(phase, other.phase) &&
+            Objects.equals(action, other.action) &&
+            Objects.equals(step, other.step) &&
+            Objects.equals(failedStep, other.failedStep) &&
+            Objects.equals(phaseTime, other.phaseTime) &&
+            Objects.equals(actionTime, other.actionTime) &&
+            Objects.equals(stepTime, other.stepTime) &&
+            Objects.equals(stepInfo, other.stepInfo) &&
+            Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+}
+
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java
new file mode 100644
index 0000000000000..22935f197731c
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleNamedXContentProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class IndexLifecycleNamedXContentProvider implements NamedXContentProvider {
+
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+            // ILM
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(AllocateAction.NAME),
+                AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(DeleteAction.NAME),
+                DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(ForceMergeAction.NAME),
+                ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(ReadOnlyAction.NAME),
+                ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(RolloverAction.NAME),
+                RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class,
+                new ParseField(ShrinkAction.NAME),
+                ShrinkAction::parse)
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoResponseTests.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleAction.java
similarity index 78%
rename from client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoResponseTests.java
rename to client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleAction.java
index 8106043c08b3a..3787d26f5f889 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoResponseTests.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleAction.java
@@ -16,11 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.client.indexlifecycle;
 
-package org.elasticsearch.client.migration;
-
-import org.elasticsearch.test.ESTestCase;
+/**
+ * interface for index lifecycle management actions
+ */
+public interface LifecycleAction {
 
-public class IndexUpgradeInfoResponseTests extends ESTestCase {
-    // TODO: add to cross XPack-HLRC serialization test
+    /**
+     * @return the name of this action
+     */
+    String getName();
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusRequest.java
new file mode 100644
index 0000000000000..5db3d2d8c4e11
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusRequest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+
+/**
+ * A {@link TimedRequest} to get the current status of index lifecycle management.
+ */
+public class LifecycleManagementStatusRequest extends TimedRequest {
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java
new file mode 100644
index 0000000000000..c1586d7e1c738
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.Objects;
+
+/**
+ * The current status of index lifecycle management. See {@link OperationMode} for available statuses.
+ */
+public class LifecycleManagementStatusResponse {
+
+    private final OperationMode operationMode;
+    private static final String OPERATION_MODE = "operation_mode";
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<LifecycleManagementStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
+        OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE));
+    }
+
+    //package private for testing
+    LifecycleManagementStatusResponse(String operationMode) {
+        this.operationMode = OperationMode.fromString(operationMode);
+    }
+
+    public OperationMode getOperationMode() {
+        return operationMode;
+    }
+
+    public static LifecycleManagementStatusResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        LifecycleManagementStatusResponse that = (LifecycleManagementStatusResponse) o;
+        return operationMode == that.operationMode;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(operationMode);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java
new file mode 100644
index 0000000000000..2dc4e3644d1e4
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Represents the lifecycle of an index from creation to deletion. A
+ * {@link LifecyclePolicy} is made up of a set of {@link Phase}s which it will
+ * move through.
+ */
+public class LifecyclePolicy implements ToXContentObject {
+    static final ParseField PHASES_FIELD = new ParseField("phases");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
+        (a, name) -> {
+            List<Phase> phases = (List<Phase>) a[0];
+            Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));
+            return new LifecyclePolicy(name, phaseMap);
+        });
+    private static Map<String, Set<String>> ALLOWED_ACTIONS = new HashMap<>();
+
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> Phase.parse(p, n), v -> {
+            throw new IllegalArgumentException("ordered " + PHASES_FIELD.getPreferredName() + " are not supported");
+        }, PHASES_FIELD);
+
+        ALLOWED_ACTIONS.put("hot", Sets.newHashSet(RolloverAction.NAME));
+        ALLOWED_ACTIONS.put("warm", Sets.newHashSet(AllocateAction.NAME, ForceMergeAction.NAME, ReadOnlyAction.NAME, ShrinkAction.NAME));
+        ALLOWED_ACTIONS.put("cold", Sets.newHashSet(AllocateAction.NAME));
+        ALLOWED_ACTIONS.put("delete", Sets.newHashSet(DeleteAction.NAME));
+    }
+
+    private final String name;
+    private final Map<String, Phase> phases;
+
+    /**
+     * @param name
+     *            the name of this {@link LifecyclePolicy}
+     * @param phases
+     *            a {@link Map} of {@link Phase}s which make up this
+     *            {@link LifecyclePolicy}.
+     */
+    public LifecyclePolicy(String name, Map<String, Phase> phases) {
+       phases.values().forEach(phase -> {
+            if (ALLOWED_ACTIONS.containsKey(phase.getName()) == false) {
+                throw new IllegalArgumentException("Lifecycle does not support phase [" + phase.getName() + "]");
+            }
+            phase.getActions().forEach((actionName, action) -> {
+                if (ALLOWED_ACTIONS.get(phase.getName()).contains(actionName) == false) {
+                    throw new IllegalArgumentException("invalid action [" + actionName + "] " +
+                        "defined in phase [" + phase.getName() +"]");
+                }
+            });
+        });
+        this.name = name;
+        this.phases = phases;
+    }
+
+    public static LifecyclePolicy parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    /**
+     * @return the name of this {@link LifecyclePolicy}
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return the {@link Phase}s for this {@link LifecyclePolicy} in the order
+     *         in which they will be executed.
+     */
+    public Map<String, Phase> getPhases() {
+        return phases;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startObject(PHASES_FIELD.getPreferredName());
+        for (Phase phase : phases.values()) {
+            builder.field(phase.getName(), phase);
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, phases);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        LifecyclePolicy other = (LifecyclePolicy) obj;
+        return Objects.equals(name, other.name) &&
+            Objects.equals(phases, other.phases);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java
new file mode 100644
index 0000000000000..84de81437065d
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Objects;
+
+public class LifecyclePolicyMetadata implements ToXContentObject {
+
+    static final ParseField POLICY = new ParseField("policy");
+    static final ParseField VERSION = new ParseField("version");
+    static final ParseField MODIFIED_DATE = new ParseField("modified_date");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
+            a -> {
+                LifecyclePolicy policy = (LifecyclePolicy) a[0];
+                return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli());
+            });
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), LifecyclePolicy::parse, POLICY);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), MODIFIED_DATE);
+    }
+
+    public static LifecyclePolicyMetadata parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final LifecyclePolicy policy;
+    private final long version;
+    private final long modifiedDate;
+
+    public LifecyclePolicyMetadata(LifecyclePolicy policy, long version, long modifiedDate) {
+        this.policy = policy;
+        this.version = version;
+        this.modifiedDate = modifiedDate;
+    }
+
+    public LifecyclePolicy getPolicy() {
+        return policy;
+    }
+
+    public String getName() {
+        return policy.getName();
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public long getModifiedDate() {
+        return modifiedDate;
+    }
+
+    public String getModifiedDateString() {
+        ZonedDateTime modifiedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(modifiedDate), ZoneOffset.UTC);
+        return modifiedDateTime.toString();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(POLICY.getPreferredName(), policy);
+        builder.field(VERSION.getPreferredName(), version);
+        builder.field(MODIFIED_DATE.getPreferredName(),
+            ZonedDateTime.ofInstant(Instant.ofEpochMilli(modifiedDate), ZoneOffset.UTC).toString());
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(policy, version, modifiedDate);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        LifecyclePolicyMetadata other = (LifecyclePolicyMetadata) obj;
+        return Objects.equals(policy, other.policy) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(modifiedDate, other.modifiedDate);
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/OperationMode.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/OperationMode.java
new file mode 100644
index 0000000000000..81634e5824ec8
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/OperationMode.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.action.admin.indices.shrink.ShrinkAction;
+
+import java.util.EnumSet;
+import java.util.Locale;
+
+/**
+ * Enum representing the different modes that Index Lifecycle Service can operate in.
+ */
+public enum OperationMode {
+    /**
+     * This represents a state where no policies are executed
+     */
+    STOPPED {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == RUNNING;
+        }
+    },
+
+    /**
+     * this represents a state where only sensitive actions (like {@link ShrinkAction}) will be executed
+     * until they finish, at which point the operation mode will move to <code>STOPPED</code>.
+     */
+    STOPPING {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == RUNNING || nextMode == STOPPED;
+        }
+    },
+
+    /**
+     * Normal operation where all policies are executed as normal.
+     */
+    RUNNING {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == STOPPING;
+        }
+    };
+
+    public abstract boolean isValidChange(OperationMode nextMode);
+
+    static OperationMode fromString(String string) {
+        return EnumSet.allOf(OperationMode.class).stream()
+            .filter(e -> string.equalsIgnoreCase(e.name())).findFirst()
+            .orElseThrow(() -> new IllegalArgumentException(String.format(Locale.ROOT, "%s is not a valid operation_mode", string)));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java
new file mode 100644
index 0000000000000..0c19d39c85964
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Represents set of {@link LifecycleAction}s which should be executed at a
+ * particular point in the lifecycle of an index.
+ */
+public class Phase implements ToXContentObject {
+
+    static final ParseField MIN_AGE = new ParseField("min_age");
+    static final ParseField ACTIONS_FIELD = new ParseField("actions");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", false,
+        (a, name) -> new Phase(name, (TimeValue) a[0], ((List<LifecycleAction>) a[1]).stream()
+            .collect(Collectors.toMap(LifecycleAction::getName, Function.identity()))));
+    static {
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> TimeValue.parseTimeValue(p.text(), MIN_AGE.getPreferredName()), MIN_AGE, ValueType.VALUE);
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(),
+            (p, c, n) -> p.namedObject(LifecycleAction.class, n, null), v -> {
+                throw new IllegalArgumentException("ordered " + ACTIONS_FIELD.getPreferredName() + " are not supported");
+            }, ACTIONS_FIELD);
+    }
+
+    public static Phase parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final String name;
+    private final Map<String, LifecycleAction> actions;
+    private final TimeValue minimumAge;
+
+    /**
+     * @param name
+     *            the name of this {@link Phase}.
+     * @param minimumAge
+     *            the age of the index when the index should move to this
+     *            {@link Phase}.
+     * @param actions
+     *            a {@link Map} of the {@link LifecycleAction}s to run when
+     *            during this {@link Phase}. The keys in this map are the associated
+     *            action names.
+     */
+    public Phase(String name, TimeValue minimumAge, Map<String, LifecycleAction> actions) {
+        this.name = name;
+        if (minimumAge == null) {
+            this.minimumAge = TimeValue.ZERO;
+        } else {
+            this.minimumAge = minimumAge;
+        }
+        this.actions = actions;
+    }
+
+    /**
+     * @return the age of the index when the index should move to this
+     *         {@link Phase}.
+     */
+    public TimeValue getMinimumAge() {
+        return minimumAge;
+    }
+
+    /**
+     * @return the name of this {@link Phase}
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return a {@link Map} of the {@link LifecycleAction}s to run when during
+     *         this {@link Phase}.
+     */
+    public Map<String, LifecycleAction> getActions() {
+        return actions;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(MIN_AGE.getPreferredName(), minimumAge.getStringRep());
+        builder.field(ACTIONS_FIELD.getPreferredName(), actions);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, minimumAge, actions);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        Phase other = (Phase) obj;
+        return Objects.equals(name, other.name) &&
+            Objects.equals(minimumAge, other.minimumAge) &&
+            Objects.equals(actions, other.actions);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java
new file mode 100644
index 0000000000000..802ca8834cdd3
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * This class contains information about the current phase being executed by Index
+ * Lifecycle Management on the specific index.
+ */
+public class PhaseExecutionInfo implements ToXContentObject {
+    private static final ParseField POLICY_NAME_FIELD = new ParseField("policy");
+    private static final ParseField PHASE_DEFINITION_FIELD = new ParseField("phase_definition");
+    private static final ParseField VERSION_FIELD = new ParseField("version");
+    private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis");
+
+    private static final ConstructingObjectParser<PhaseExecutionInfo, String> PARSER = new ConstructingObjectParser<>(
+        "phase_execution_info", false,
+        (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3]));
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Phase::parse, PHASE_DEFINITION_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_IN_MILLIS_FIELD);
+    }
+
+    public static PhaseExecutionInfo parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final String policyName;
+    private final Phase phase;
+    private final long version;
+    private final long modifiedDate;
+
+    /**
+     * This class holds information about the current phase that is being executed
+     *
+     * @param policyName the name of the policy being executed, this may not be the current policy assigned to an index
+     * @param phase the current phase definition executed
+     * @param version the version of the <code>policyName</code> being executed
+     * @param modifiedDate the time the executing version of the phase was modified
+     */
+    public PhaseExecutionInfo(String policyName, Phase phase, long version, long modifiedDate) {
+        this.policyName = policyName;
+        this.phase = phase;
+        this.version = version;
+        this.modifiedDate = modifiedDate;
+    }
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    public Phase getPhase() {
+        return phase;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public long getModifiedDate() {
+        return modifiedDate;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(policyName, phase, version, modifiedDate);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        PhaseExecutionInfo other = (PhaseExecutionInfo) obj;
+        return Objects.equals(policyName, other.policyName) &&
+            Objects.equals(phase, other.phase) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(modifiedDate, other.modifiedDate);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, false, true);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
+        if (phase != null) {
+            builder.field(PHASE_DEFINITION_FIELD.getPreferredName(), phase);
+        }
+        builder.field(VERSION_FIELD.getPreferredName(), version);
+        builder.timeField(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate);
+        builder.endObject();
+        return builder;
+    }
+}
+
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequest.java
new file mode 100644
index 0000000000000..ddfcc6bf6e65a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutLifecyclePolicyRequest extends TimedRequest implements ToXContentObject {
+
+    private final LifecyclePolicy policy;
+
+    public PutLifecyclePolicyRequest(LifecyclePolicy policy) {
+        if (policy == null) {
+            throw new IllegalArgumentException("policy definition cannot be null");
+        }
+        if (Strings.isNullOrEmpty(policy.getName())) {
+            throw new IllegalArgumentException("policy name must be present");
+        }
+        this.policy = policy;
+    }
+
+    public String getName() {
+        return policy.getName();
+    }
+
+    public LifecyclePolicy getLifecyclePolicy() {
+        return policy;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("policy", policy);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PutLifecyclePolicyRequest that = (PutLifecyclePolicyRequest) o;
+        return Objects.equals(getLifecyclePolicy(), that.getLifecyclePolicy());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getLifecyclePolicy());
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java
new file mode 100644
index 0000000000000..7734e792bbc5b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class ReadOnlyAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "readonly";
+
+    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
+
+    public static ReadOnlyAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ReadOnlyAction() {
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return ReadOnlyAction.class.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequest.java
new file mode 100644
index 0000000000000..88bdf4dd6868d
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.TimedRequest;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class RemoveIndexLifecyclePolicyRequest extends TimedRequest {
+
+    private final List<String> indices;
+    private final IndicesOptions indicesOptions;
+
+    public RemoveIndexLifecyclePolicyRequest(List<String> indices) {
+        this(indices, IndicesOptions.strictExpandOpen());
+    }
+
+    public RemoveIndexLifecyclePolicyRequest(List<String> indices, IndicesOptions indicesOptions) {
+        this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices));
+        this.indicesOptions = Objects.requireNonNull(indicesOptions);
+    }
+
+    public List<String> indices() {
+        return indices;
+    }
+
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indices, indicesOptions);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        RemoveIndexLifecyclePolicyRequest other = (RemoveIndexLifecyclePolicyRequest) obj;
+        return Objects.deepEquals(indices, other.indices) &&
+                Objects.equals(indicesOptions, other.indicesOptions);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponse.java
new file mode 100644
index 0000000000000..3aae1537faa29
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponse.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class RemoveIndexLifecyclePolicyResponse {
+
+    public static final ParseField HAS_FAILURES_FIELD = new ParseField("has_failures");
+    public static final ParseField FAILED_INDEXES_FIELD = new ParseField("failed_indexes");
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<RemoveIndexLifecyclePolicyResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "change_policy_for_index_response", true, args -> new RemoveIndexLifecyclePolicyResponse((List<String>)args[0]));
+    static {
+        PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), FAILED_INDEXES_FIELD);
+        // Needs to be declared but not used in constructing the response object
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), HAS_FAILURES_FIELD);
+    }
+
+    private final List<String> failedIndexes;
+
+    public RemoveIndexLifecyclePolicyResponse(List<String> failedIndexes) {
+        if (failedIndexes == null) {
+            throw new IllegalArgumentException(FAILED_INDEXES_FIELD.getPreferredName() + " cannot be null");
+        }
+        this.failedIndexes = Collections.unmodifiableList(failedIndexes);
+    }
+
+    public List<String> getFailedIndexes() {
+        return failedIndexes;
+    }
+
+    public boolean hasFailures() {
+        return failedIndexes.isEmpty() == false;
+    }
+
+    public static RemoveIndexLifecyclePolicyResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(failedIndexes);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        RemoveIndexLifecyclePolicyResponse other = (RemoveIndexLifecyclePolicyResponse) obj;
+        return Objects.equals(failedIndexes, other.failedIndexes);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RetryLifecyclePolicyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RetryLifecyclePolicyRequest.java
new file mode 100644
index 0000000000000..6f3acaf19aaea
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RetryLifecyclePolicyRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import org.elasticsearch.client.TimedRequest;
+
+public class RetryLifecyclePolicyRequest extends TimedRequest {
+
+    private final List<String> indices;
+
+    public RetryLifecyclePolicyRequest(String... indices) {
+        if (indices.length == 0) {
+            throw new IllegalArgumentException("Must at least specify one index to retry");
+        }
+        this.indices = Arrays.asList(indices);
+    }
+
+    public List<String> getIndices() {
+        return indices;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        RetryLifecyclePolicyRequest that = (RetryLifecyclePolicyRequest) o;
+        return indices.size() == that.indices.size() && indices.containsAll(that.indices);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indices);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java
new file mode 100644
index 0000000000000..0cc9dcf234969
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+
+public class RolloverAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "rollover";
+    private static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
+    private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
+    private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
+
+    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+        a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
+    static {
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
+    }
+
+    private final ByteSizeValue maxSize;
+    private final Long maxDocs;
+    private final TimeValue maxAge;
+
+    public static RolloverAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public RolloverAction(ByteSizeValue maxSize, TimeValue maxAge, Long maxDocs) {
+        if (maxSize == null && maxAge == null && maxDocs == null) {
+            throw new IllegalArgumentException("At least one rollover condition must be set.");
+        }
+        this.maxSize = maxSize;
+        this.maxAge = maxAge;
+        this.maxDocs = maxDocs;
+    }
+    public ByteSizeValue getMaxSize() {
+        return maxSize;
+    }
+
+    public TimeValue getMaxAge() {
+        return maxAge;
+    }
+
+    public Long getMaxDocs() {
+        return maxDocs;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (maxSize != null) {
+            builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
+        }
+        if (maxAge != null) {
+            builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
+        }
+        if (maxDocs != null) {
+            builder.field(MAX_DOCS_FIELD.getPreferredName(), maxDocs);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        RolloverAction other = (RolloverAction) obj;
+        return Objects.equals(maxSize, other.maxSize) &&
+            Objects.equals(maxAge, other.maxAge) &&
+            Objects.equals(maxDocs, other.maxDocs);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java
new file mode 100644
index 0000000000000..345356380145e
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ShrinkAction implements LifecycleAction, ToXContentObject {
+    public static final String NAME = "shrink";
+    private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards");
+
+    private static final ConstructingObjectParser<ShrinkAction, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD);
+    }
+
+    private int numberOfShards;
+
+    public static ShrinkAction parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public ShrinkAction(int numberOfShards) {
+        if (numberOfShards <= 0) {
+            throw new IllegalArgumentException("[" + NUMBER_OF_SHARDS_FIELD.getPreferredName() + "] must be greater than 0");
+        }
+        this.numberOfShards = numberOfShards;
+    }
+
+    int getNumberOfShards() {
+        return numberOfShards;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(NUMBER_OF_SHARDS_FIELD.getPreferredName(), numberOfShards);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ShrinkAction that = (ShrinkAction) o;
+        return Objects.equals(numberOfShards, that.numberOfShards);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(numberOfShards);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StartILMRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StartILMRequest.java
new file mode 100644
index 0000000000000..84cc844a92a98
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StartILMRequest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+
+public class StartILMRequest extends TimedRequest {
+
+    public StartILMRequest() {
+    }
+
+    @Override
+    public int hashCode() {
+        return 64;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StopILMRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StopILMRequest.java
new file mode 100644
index 0000000000000..1695fc0dd7aea
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/StopILMRequest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.client.TimedRequest;
+
+public class StopILMRequest extends TimedRequest {
+
+    public StopILMRequest() {
+    }
+
+    @Override
+    public int hashCode() {
+        return 75;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/license/StartBasicResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/license/StartBasicResponse.java
index f6ab026402462..c2596f3e38a4e 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/license/StartBasicResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/license/StartBasicResponse.java
@@ -84,14 +84,13 @@ public class StartBasicResponse {
                     }
                 }
                 return new Tuple<>(message, acknowledgeMessages);
-            },
-            new ParseField("acknowledge"));
+            }, new ParseField("acknowledge"));
     }
 
     private Map<String, String[]> acknowledgeMessages;
     private String acknowledgeMessage;
 
-    enum Status {
+    public enum Status {
         GENERATED_BASIC(true, null, RestStatus.OK),
         ALREADY_USING_BASIC(false, "Operation failed: Current license is basic.", RestStatus.FORBIDDEN),
         NEED_ACKNOWLEDGEMENT(false, "Operation failed: Needs acknowledgement.", RestStatus.OK);
@@ -141,6 +140,10 @@ public StartBasicResponse() {
         this.acknowledgeMessage = acknowledgeMessage;
     }
 
+    public Status getStatus() {
+        return status;
+    }
+
     public boolean isAcknowledged() {
         return status != StartBasicResponse.Status.NEED_ACKNOWLEDGEMENT;
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/UpgradeActionRequired.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/UpgradeActionRequired.java
index e743d10529e1a..26b7b1e815d7f 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/UpgradeActionRequired.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/migration/UpgradeActionRequired.java
@@ -18,17 +18,12 @@
  */
 package org.elasticsearch.client.migration;
 
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
 import java.util.Locale;
 
 /**
  * Indicates the type of the upgrade required for the index
  */
-public enum UpgradeActionRequired implements Writeable {
+public enum UpgradeActionRequired {
     NOT_APPLICABLE,   // Indicates that the check is not applicable to this index type, the next check will be performed
     UP_TO_DATE,       // Indicates that the check finds this index to be up to date - no additional checks are required
     REINDEX,          // The index should be reindex
@@ -38,15 +33,6 @@ public static UpgradeActionRequired fromString(String value) {
         return UpgradeActionRequired.valueOf(value.toUpperCase(Locale.ROOT));
     }
 
-    public static UpgradeActionRequired readFromStream(StreamInput in) throws IOException {
-        return in.readEnum(UpgradeActionRequired.class);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeEnum(this);
-    }
-
     @Override
     public String toString() {
         return name().toLowerCase(Locale.ROOT);
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterRequest.java
new file mode 100644
index 0000000000000..5414c86258111
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterRequest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.client.ml.job.config.MlFilter;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Request to create a new Machine Learning MlFilter given a {@link MlFilter} configuration
+ */
+public class PutFilterRequest extends ActionRequest implements ToXContentObject {
+
+    private final MlFilter filter;
+
+    /**
+     * Construct a new PutMlFilterRequest
+     *
+     * @param filter a {@link MlFilter} configuration to create
+     */
+    public PutFilterRequest(MlFilter filter) {
+        this.filter = filter;
+    }
+
+    public MlFilter getMlFilter() {
+        return filter;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return filter.toXContent(builder, params);
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+
+        PutFilterRequest request = (PutFilterRequest) object;
+        return Objects.equals(filter, request.filter);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(filter);
+    }
+
+    @Override
+    public final String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterResponse.java
new file mode 100644
index 0000000000000..56164bd5be08e
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutFilterResponse.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.job.config.MlFilter;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Response containing the newly created {@link MlFilter}
+ */
+public class PutFilterResponse implements ToXContentObject {
+
+    private MlFilter filter;
+
+    public static PutFilterResponse fromXContent(XContentParser parser) throws IOException {
+        return new PutFilterResponse(MlFilter.PARSER.parse(parser, null).build());
+    }
+
+    PutFilterResponse(MlFilter filter) {
+        this.filter = filter;
+    }
+
+    public MlFilter getResponse() {
+        return filter;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        filter.toXContent(builder, params);
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+        PutFilterResponse response = (PutFilterResponse) object;
+        return Objects.equals(filter, response.filter);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(filter);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/MlFilter.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/MlFilter.java
index e0d1bd0849b3b..1b67fc4459b50 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/MlFilter.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/MlFilter.java
@@ -32,6 +32,14 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+/**
+ * An MlFilter Object
+ *
+ * A filter contains a list of strings.
+ * It can be used by one or more jobs.
+ *
+ * Specifically, filters are referenced in the custom_rules property of detector configuration objects.
+ */
 public class MlFilter implements ToXContentObject {
 
     public static final ParseField TYPE = new ParseField("type");
@@ -105,6 +113,10 @@ public int hashCode() {
         return Objects.hash(id, description, items);
     }
 
+    /**
+     * Creates a new Builder object for creating an MlFilter object
+     * @param filterId The ID of the filter to create
+     */
     public static Builder builder(String filterId) {
         return new Builder().setId(filterId);
     }
@@ -118,6 +130,10 @@ public static class Builder {
         private Builder() {
         }
 
+        /**
+         * Set the ID of the filter
+         * @param id The id desired
+         */
         public Builder setId(String id) {
             this.id = Objects.requireNonNull(id);
             return this;
@@ -128,6 +144,10 @@ public String getId() {
             return id;
         }
 
+        /**
+         * Set the description of the filter
+         * @param description The description desired
+         */
         public Builder setDescription(String description) {
             this.description = description;
             return this;
@@ -143,6 +163,13 @@ public Builder setItems(List<String> items) {
             return this;
         }
 
+        /**
+         * The items of the filter.
+         *
+         * A wildcard * can be used at the beginning or the end of an item. Up to 10000 items are allowed in each filter.
+         *
+         * @param items String list of items to be applied in the filter
+         */
         public Builder setItems(String... items) {
             setItems(Arrays.asList(items));
             return this;
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/AcknowledgedResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/AcknowledgedResponse.java
index 4e279844afc59..8a588b24d536c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/AcknowledgedResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/AcknowledgedResponse.java
@@ -32,9 +32,9 @@
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
 public abstract class AcknowledgedResponse implements ToXContentObject {
+    private final boolean acknowledged;
 
     protected static final String PARSE_FIELD_NAME = "acknowledged";
-    private final boolean acknowledged;
 
     public AcknowledgedResponse(final boolean acknowledged) {
         this.acknowledged = acknowledged;
@@ -83,4 +83,5 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
     protected String getFieldName() {
         return PARSE_FIELD_NAME;
     }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobRequest.java
new file mode 100644
index 0000000000000..9b7a322b23827
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobRequest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.rollup;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+
+public class DeleteRollupJobRequest implements Validatable, ToXContentObject {
+
+    private static final ParseField ID_FIELD = new ParseField("id");
+    private final String id;
+
+
+    public DeleteRollupJobRequest(String id) {
+        this.id = Objects.requireNonNull(id, "id parameter must not be null");
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    private static final ConstructingObjectParser<DeleteRollupJobRequest, Void> PARSER =
+        new ConstructingObjectParser<>("request",  a -> {
+            return new DeleteRollupJobRequest((String) a[0]);
+        });
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD);
+    }
+
+    public static DeleteRollupJobRequest fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID_FIELD.getPreferredName(), this.id);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DeleteRollupJobRequest that = (DeleteRollupJobRequest) o;
+        return Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobResponse.java
new file mode 100644
index 0000000000000..adba15aaca552
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/DeleteRollupJobResponse.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.rollup;
+
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class DeleteRollupJobResponse extends AcknowledgedResponse {
+
+    public DeleteRollupJobResponse(boolean acknowledged) {
+        super(acknowledged);
+    }
+
+    private static final ConstructingObjectParser<DeleteRollupJobResponse, Void> PARSER = AcknowledgedResponse
+        .generateParser("delete_rollup_job_response", DeleteRollupJobResponse::new, AcknowledgedResponse.PARSE_FIELD_NAME);
+
+    public static DeleteRollupJobResponse fromXContent(final XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/PutRollupJobResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/PutRollupJobResponse.java
index 60fc2842100cf..ea26343c76635 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/PutRollupJobResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/PutRollupJobResponse.java
@@ -23,17 +23,18 @@
 
 import java.io.IOException;
 
-
 public class PutRollupJobResponse extends AcknowledgedResponse {
 
+
     public PutRollupJobResponse(boolean acknowledged) {
         super(acknowledged);
     }
 
     private static final ConstructingObjectParser<PutRollupJobResponse, Void> PARSER = AcknowledgedResponse
-            .generateParser("delete_rollup_job_response", PutRollupJobResponse::new, AcknowledgedResponse.PARSE_FIELD_NAME);
+        .generateParser("delete_rollup_job_response", PutRollupJobResponse::new, AcknowledgedResponse.PARSE_FIELD_NAME);
 
     public static PutRollupJobResponse fromXContent(final XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateRequest.java
new file mode 100644
index 0000000000000..2aefa97cb8bf1
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateRequest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Validatable;
+
+/**
+ * Empty request object required to make the authenticate call. The authenticate call
+ * retrieves metadata about the authenticated user.
+ */
+public final class AuthenticateRequest implements Validatable {
+
+    public static final AuthenticateRequest INSTANCE = new AuthenticateRequest();
+
+    private AuthenticateRequest() {
+    }
+
+    public Request getRequest() {
+        return new Request(HttpGet.METHOD_NAME, "/_xpack/security/_authenticate");
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateResponse.java
new file mode 100644
index 0000000000000..62f1cc0955bd1
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateResponse.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.security.user.User;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * The response for the authenticate call. The response contains two fields: a
+ * user field and a boolean flag signaling if the user is enabled or not. The
+ * user object contains all user metadata which Elasticsearch uses to map roles,
+ * etc.
+ */
+public final class AuthenticateResponse {
+
+    static final ParseField USERNAME = new ParseField("username");
+    static final ParseField ROLES = new ParseField("roles");
+    static final ParseField METADATA = new ParseField("metadata");
+    static final ParseField FULL_NAME = new ParseField("full_name");
+    static final ParseField EMAIL = new ParseField("email");
+    static final ParseField ENABLED = new ParseField("enabled");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<AuthenticateResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "client_security_authenticate_response",
+            a -> new AuthenticateResponse(new User((String) a[0], ((List<String>) a[1]), (Map<String, Object>) a[2],
+                    (String) a[3], (String) a[4]), (Boolean) a[5]));
+    static {
+        PARSER.declareString(constructorArg(), USERNAME);
+        PARSER.declareStringArray(constructorArg(), ROLES);
+        PARSER.<Map<String, Object>>declareObject(constructorArg(), (parser, c) -> parser.map(), METADATA);
+        PARSER.declareStringOrNull(optionalConstructorArg(), FULL_NAME);
+        PARSER.declareStringOrNull(optionalConstructorArg(), EMAIL);
+        PARSER.declareBoolean(constructorArg(), ENABLED);
+    }
+
+    private final User user;
+    private final boolean enabled;
+
+    public AuthenticateResponse(User user, boolean enabled) {
+        this.user = user;
+        this.enabled = enabled;
+    }
+
+    /**
+     * @return The effective user. This is the authenticated user, or, when
+     *         submitting requests on behalf of other users, it is the
+     *         impersonated user.
+     */
+    public User getUser() {
+        return user;
+    }
+
+    /**
+     * @return whether the user is enabled or not
+     */
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final AuthenticateResponse that = (AuthenticateResponse) o;
+        return user.equals(that.user) && enabled == that.enabled;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(user, enabled);
+    }
+
+    public static AuthenticateResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheRequest.java
new file mode 100644
index 0000000000000..268fc4a1de6e0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheRequest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Request for clearing the cache of one or more realms
+ */
+public final class ClearRealmCacheRequest implements Validatable {
+
+    private final List<String> realms;
+    private final List<String> usernames;
+
+    /**
+     * Create a new request to clear cache of realms
+     * @param realms the realms to clear the cache of. Must not be {@code null}. An empty list
+     *               indicates that all realms should have their caches cleared.
+     * @param usernames the usernames to clear the cache of. Must not be {@code null}. An empty
+     *                  list indicates that every user in the listed realms should have their cache
+     *                  cleared.
+     */
+    public ClearRealmCacheRequest(List<String> realms, List<String> usernames) {
+        this.realms = Collections.unmodifiableList(Objects.requireNonNull(realms, "the realms list must not be null"));
+        this.usernames = Collections.unmodifiableList(Objects.requireNonNull(usernames, "usernames list must no be null"));
+    }
+
+    public List<String> getRealms() {
+        return realms;
+    }
+
+    public List<String> getUsernames() {
+        return usernames;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheResponse.java
new file mode 100644
index 0000000000000..ce1495f9ef2b8
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRealmCacheResponse.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.NodesResponseHeader;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Response for a clear realm cache request. The response includes a header that contains the
+ * number of successful and failed nodes.
+ */
+public final class ClearRealmCacheResponse extends SecurityNodesResponse {
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<ClearRealmCacheResponse, Void> PARSER =
+        new ConstructingObjectParser<>("clear_realm_cache_response_parser",
+            args -> new ClearRealmCacheResponse((List<Node>) args[0], (NodesResponseHeader) args[1], (String) args[2]));
+
+    static {
+        SecurityNodesResponse.declareCommonNodesResponseParsing(PARSER);
+    }
+
+    public ClearRealmCacheResponse(List<Node> nodes, NodesResponseHeader header, String clusterName) {
+        super(nodes, header, clusterName);
+    }
+
+    public static ClearRealmCacheResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRolesCacheResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRolesCacheResponse.java
index b6b864a37e226..c7df7e0f492e4 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRolesCacheResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/ClearRolesCacheResponse.java
@@ -20,18 +20,16 @@
 package org.elasticsearch.client.security;
 
 import org.elasticsearch.client.NodesResponseHeader;
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Objects;
 
 /**
- * The response object that will be returned when clearing the cache of native roles
+ * The response object that will be returned when clearing the roles cache
  */
-public final class ClearRolesCacheResponse {
+public final class ClearRolesCacheResponse extends SecurityNodesResponse {
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<ClearRolesCacheResponse, Void> PARSER =
@@ -39,68 +37,11 @@ public final class ClearRolesCacheResponse {
             args -> new ClearRolesCacheResponse((List<Node>)args[0], (NodesResponseHeader) args[1], (String) args[2]));
 
     static {
-        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> Node.PARSER.apply(p, n),
-            new ParseField("nodes"));
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), NodesResponseHeader::fromXContent, new ParseField("_nodes"));
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("cluster_name"));
+        SecurityNodesResponse.declareCommonNodesResponseParsing(PARSER);
     }
 
-    private final List<Node> nodes;
-    private final NodesResponseHeader header;
-    private final String clusterName;
-
     public ClearRolesCacheResponse(List<Node> nodes, NodesResponseHeader header, String clusterName) {
-        this.nodes = nodes;
-        this.header = header;
-        this.clusterName = Objects.requireNonNull(clusterName, "cluster name must be provided");
-    }
-
-    /** returns a list of nodes in which the cache was cleared */
-    public List<Node> getNodes() {
-        return nodes;
-    }
-
-    /**
-     * Get the cluster name associated with all of the nodes.
-     *
-     * @return Never {@code null}.
-     */
-    public String getClusterName() {
-        return clusterName;
-    }
-
-    /**
-     * Gets information about the number of total, successful and failed nodes the request was run on.
-     * Also includes exceptions if relevant.
-     */
-    public NodesResponseHeader getHeader() {
-        return header;
-    }
-
-    public static class Node {
-
-        private static final ConstructingObjectParser<Node, String> PARSER =
-            new ConstructingObjectParser<>("clear_roles_cache_response_node", false, (args, id) -> new Node(id, (String) args[0]));
-
-        static {
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("name"));
-        }
-
-        private final String id;
-        private final String name;
-
-        public Node(String id, String name) {
-            this.id = id;
-            this.name = name;
-        }
-
-        public String getId() {
-            return id;
-        }
-
-        public String getName() {
-            return name;
-        }
+        super(nodes, header, clusterName);
     }
 
     public static ClearRolesCacheResponse fromXContent(XContentParser parser) throws IOException {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/SecurityNodesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/SecurityNodesResponse.java
new file mode 100644
index 0000000000000..22b9e8220e743
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/SecurityNodesResponse.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.NodesResponse;
+import org.elasticsearch.client.NodesResponseHeader;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+
+import java.util.List;
+
+/**
+ * Base class for security responses that are node responses. Security uses a common pattern in the
+ * response so this class is present to avoid duplication.
+ */
+public abstract class SecurityNodesResponse extends NodesResponse {
+
+    private final List<Node> nodes;
+
+    SecurityNodesResponse(List<Node> nodes, NodesResponseHeader header, String clusterName) {
+        super(header, clusterName);
+        this.nodes = nodes;
+    }
+
+    /** returns a list of nodes in which the cache was cleared */
+    public List<Node> getNodes() {
+        return nodes;
+    }
+
+    public static class Node {
+
+        private static final ConstructingObjectParser<ClearRolesCacheResponse.Node, String> PARSER =
+            new ConstructingObjectParser<>("clear_roles_cache_response_node", false,
+                (args, id) -> new ClearRolesCacheResponse.Node(id, (String) args[0]));
+
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("name"));
+        }
+
+        private final String id;
+        private final String name;
+
+        public Node(String id, String name) {
+            this.id = id;
+            this.name = name;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+
+    public static <T extends NodesResponse> void declareCommonNodesResponseParsing(ConstructingObjectParser<T, Void> parser) {
+        parser.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> Node.PARSER.apply(p, n),
+            new ParseField("nodes"));
+        NodesResponse.declareCommonNodesResponseParsing(parser);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/User.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/User.java
new file mode 100644
index 0000000000000..977780b46b79b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/User.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security.user;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+
+/**
+ * An authenticated user
+ */
+public final class User {
+
+    private final String username;
+    private final Collection<String> roles;
+    private final Map<String, Object> metadata;
+    @Nullable private final String fullName;
+    @Nullable private final String email;
+
+    public User(String username, Collection<String> roles, Map<String, Object> metadata, @Nullable String fullName,
+            @Nullable String email) {
+        Objects.requireNonNull(username, "`username` cannot be null");
+        Objects.requireNonNull(roles, "`roles` cannot be null. Pass an empty collection instead.");
+        Objects.requireNonNull(roles, "`metadata` cannot be null. Pass an empty map instead.");
+        this.username = username;
+        this.roles = roles;
+        this.metadata = Collections.unmodifiableMap(metadata);
+        this.fullName = fullName;
+        this.email = email;
+    }
+
+    /**
+     * @return  The principal of this user - effectively serving as the
+     *          unique identity of the user. Can never be {@code null}.
+     */
+    public String username() {
+        return this.username;
+    }
+
+    /**
+     * @return  The roles this user is associated with. The roles are
+     *          identified by their unique names and each represents as
+     *          set of permissions. Can never be {@code null}.
+     */
+    public Collection<String> roles() {
+        return this.roles;
+    }
+
+    /**
+     * @return  The metadata that is associated with this user. Can never be {@code null}.
+     */
+    public Map<String, Object> metadata() {
+        return metadata;
+    }
+
+    /**
+     * @return  The full name of this user. May be {@code null}.
+     */
+    public @Nullable String fullName() {
+        return fullName;
+    }
+
+    /**
+     * @return  The email of this user. May be {@code null}.
+     */
+    public @Nullable String email() {
+        return email;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("User[username=").append(username);
+        sb.append(",roles=[").append(Strings.collectionToCommaDelimitedString(roles)).append("]");
+        sb.append(",metadata=").append(metadata);
+        sb.append(",fullName=").append(fullName);
+        sb.append(",email=").append(email);
+        sb.append("]");
+        return sb.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o instanceof User == false) {
+            return false;
+        }
+
+        final User user = (User) o;
+
+        if (!username.equals(user.username)) {
+            return false;
+        }
+        if (!roles.equals(user.roles)) {
+            return false;
+        }
+        if (!metadata.equals(user.metadata)) {
+            return false;
+        }
+        if (fullName != null ? !fullName.equals(user.fullName) : user.fullName != null) {
+            return false;
+        }
+        return !(email != null ? !email.equals(user.email) : user.email != null);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(username, roles, metadata, fullName, email);
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
index fadffab95f378..f9a92d2fbbe02 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
@@ -18,13 +18,9 @@
  */
 package org.elasticsearch.client.xpack;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -253,7 +249,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     }
 
-    public static class BuildInfo implements ToXContentObject, Writeable {
+    public static class BuildInfo implements ToXContentObject {
         private final String hash;
         private final String timestamp;
 
@@ -262,16 +258,6 @@ public BuildInfo(String hash, String timestamp) {
             this.timestamp = timestamp;
         }
 
-        public BuildInfo(StreamInput input) throws IOException {
-            this(input.readString(), input.readString());
-        }
-
-        @Override
-        public void writeTo(StreamOutput output) throws IOException {
-            output.writeString(hash);
-            output.writeString(timestamp);
-        }
-
         public String getHash() {
             return hash;
         }
@@ -310,7 +296,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     }
 
-    public static class FeatureSetsInfo implements ToXContentObject, Writeable {
+    public static class FeatureSetsInfo implements ToXContentObject {
         private final Map<String, FeatureSet> featureSets;
 
         public FeatureSetsInfo(Set<FeatureSet> featureSets) {
@@ -321,24 +307,6 @@ public FeatureSetsInfo(Set<FeatureSet> featureSets) {
             this.featureSets = Collections.unmodifiableMap(map);
         }
 
-        public FeatureSetsInfo(StreamInput in) throws IOException {
-            int size = in.readVInt();
-            Map<String, FeatureSet> featureSets = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                FeatureSet featureSet = new FeatureSet(in);
-                featureSets.put(featureSet.name, featureSet);
-            }
-            this.featureSets = Collections.unmodifiableMap(featureSets);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(featureSets.size());
-            for (FeatureSet featureSet : featureSets.values()) {
-                featureSet.writeTo(out);
-            }
-        }
-
         public Map<String, FeatureSet> getFeatureSets() {
             return featureSets;
         }
@@ -366,7 +334,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             return builder.endObject();
         }
 
-        public static class FeatureSet implements ToXContentObject, Writeable {
+        public static class FeatureSet implements ToXContentObject {
             private final String name;
             @Nullable private final String description;
             private final boolean available;
@@ -382,22 +350,6 @@ public FeatureSet(String name, @Nullable String description, boolean available,
                 this.nativeCodeInfo = nativeCodeInfo;
             }
 
-            public FeatureSet(StreamInput in) throws IOException {
-                this(in.readString(), in.readOptionalString(), in.readBoolean(), in.readBoolean(),
-                        in.getVersion().onOrAfter(Version.V_5_4_0) ? in.readMap() : null);
-            }
-
-            @Override
-            public void writeTo(StreamOutput out) throws IOException {
-                out.writeString(name);
-                out.writeOptionalString(description);
-                out.writeBoolean(available);
-                out.writeBoolean(enabled);
-                if (out.getVersion().onOrAfter(Version.V_5_4_0)) {
-                    out.writeMap(nativeCodeInfo);
-                }
-            }
-
             public String name() {
                 return name;
             }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackUsageResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackUsageResponse.java
index b51a2d7de9fbd..2f9c99cc65e09 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackUsageResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackUsageResponse.java
@@ -34,7 +34,7 @@ public class XPackUsageResponse {
 
     private final Map<String, Map<String, Object>> usages;
 
-    private XPackUsageResponse(Map<String, Map<String, Object>> usages) throws IOException {
+    private XPackUsageResponse(Map<String, Map<String, Object>> usages) {
         this.usages = usages;
     }
 
diff --git a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 0000000000000..4204a868246a5
--- /dev/null
+++ b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1 @@
+org.elasticsearch.client.indexlifecycle.IndexLifecycleNamedXContentProvider
\ No newline at end of file
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
index 8dba3cafd0b44..083529ae214ef 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
@@ -77,16 +77,42 @@ protected static <Req, Resp> Resp execute(Req request, SyncMethod<Req, Resp> syn
         }
     }
 
+    /**
+     * Executes the provided request using either the sync method or its async
+     * variant, both provided as functions. This variant is used when the call does
+     * not have a request object (only headers and the request path).
+     */
+    protected static <Resp> Resp execute(SyncMethodNoRequest<Resp> syncMethodNoRequest, AsyncMethodNoRequest<Resp> asyncMethodNoRequest,
+            RequestOptions requestOptions) throws IOException {
+        if (randomBoolean()) {
+            return syncMethodNoRequest.execute(requestOptions);
+        } else {
+            PlainActionFuture<Resp> future = PlainActionFuture.newFuture();
+            asyncMethodNoRequest.execute(requestOptions, future);
+            return future.actionGet();
+        }
+    }
+
     @FunctionalInterface
     protected interface SyncMethod<Request, Response> {
         Response execute(Request request, RequestOptions options) throws IOException;
     }
 
+    @FunctionalInterface
+    protected interface SyncMethodNoRequest<Response> {
+        Response execute(RequestOptions options) throws IOException;
+    }
+
     @FunctionalInterface
     protected interface AsyncMethod<Request, Response> {
         void execute(Request request, RequestOptions options, ActionListener<Response> listener);
     }
 
+    @FunctionalInterface
+    protected interface AsyncMethodNoRequest<Response> {
+        void execute(RequestOptions options, ActionListener<Response> listener);
+    }
+
     /**
      * Executes the provided request using either the sync method or its async variant, both provided as functions
      */
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java
new file mode 100644
index 0000000000000..f2040bc88da34
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.indexlifecycle.AllocateAction;
+import org.elasticsearch.client.indexlifecycle.DeleteAction;
+import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.client.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.IndexLifecycleExplainResponse;
+import org.elasticsearch.client.indexlifecycle.LifecycleAction;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
+import org.elasticsearch.client.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.client.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.client.indexlifecycle.OperationMode;
+import org.elasticsearch.client.indexlifecycle.Phase;
+import org.elasticsearch.client.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyResponse;
+import org.elasticsearch.client.indexlifecycle.RolloverAction;
+import org.elasticsearch.client.indexlifecycle.ShrinkAction;
+import org.elasticsearch.client.indexlifecycle.StartILMRequest;
+import org.elasticsearch.client.indexlifecycle.StopILMRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.is;
+
+public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
+
+    public void testRemoveIndexLifecyclePolicy() throws Exception {
+        String policyName = randomAlphaOfLength(10);
+        LifecyclePolicy policy = createRandomPolicy(policyName);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+        assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+                highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+
+        createIndex("foo", Settings.builder().put("index.lifecycle.name", policyName).build());
+        createIndex("baz", Settings.builder().put("index.lifecycle.name", policyName).build());
+        createIndex("rbh", Settings.builder().put("index.lifecycle.name", policyName).build());
+
+        GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("foo", "baz", "rbh");
+        GetSettingsResponse settingsResponse = highLevelClient().indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT);
+        assertThat(settingsResponse.getSetting("foo", "index.lifecycle.name"), equalTo(policyName));
+        assertThat(settingsResponse.getSetting("baz", "index.lifecycle.name"), equalTo(policyName));
+        assertThat(settingsResponse.getSetting("rbh", "index.lifecycle.name"), equalTo(policyName));
+
+        List<String> indices = new ArrayList<>();
+        indices.add("foo");
+        indices.add("rbh");
+        RemoveIndexLifecyclePolicyRequest removeReq = new RemoveIndexLifecyclePolicyRequest(indices);
+        RemoveIndexLifecyclePolicyResponse removeResp = execute(removeReq, highLevelClient().indexLifecycle()::removeIndexLifecyclePolicy,
+                highLevelClient().indexLifecycle()::removeIndexLifecyclePolicyAsync);
+        assertThat(removeResp.hasFailures(), is(false));
+        assertThat(removeResp.getFailedIndexes().isEmpty(), is(true));
+
+        getSettingsRequest = new GetSettingsRequest().indices("foo", "baz", "rbh");
+        settingsResponse = highLevelClient().indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT);
+        assertNull(settingsResponse.getSetting("foo", "index.lifecycle.name"));
+        assertThat(settingsResponse.getSetting("baz", "index.lifecycle.name"), equalTo(policyName));
+        assertNull(settingsResponse.getSetting("rbh", "index.lifecycle.name"));
+    }
+
+    public void testStartStopILM() throws Exception {
+        String policyName = randomAlphaOfLength(10);
+        LifecyclePolicy policy = createRandomPolicy(policyName);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+        assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+
+        createIndex("foo", Settings.builder().put("index.lifecycle.name", "bar").build());
+        createIndex("baz", Settings.builder().put("index.lifecycle.name", "eggplant").build());
+        createIndex("squash", Settings.EMPTY);
+
+        LifecycleManagementStatusRequest statusRequest = new LifecycleManagementStatusRequest();
+        LifecycleManagementStatusResponse statusResponse = execute(
+            statusRequest,
+            highLevelClient().indexLifecycle()::lifecycleManagementStatus,
+            highLevelClient().indexLifecycle()::lifecycleManagementStatusAsync);
+        assertEquals(statusResponse.getOperationMode(), OperationMode.RUNNING);
+
+        StopILMRequest stopReq = new StopILMRequest();
+        AcknowledgedResponse stopResponse = execute(stopReq, highLevelClient().indexLifecycle()::stopILM,
+                highLevelClient().indexLifecycle()::stopILMAsync);
+        assertTrue(stopResponse.isAcknowledged());
+
+
+        statusResponse = execute(statusRequest, highLevelClient().indexLifecycle()::lifecycleManagementStatus,
+            highLevelClient().indexLifecycle()::lifecycleManagementStatusAsync);
+        assertThat(statusResponse.getOperationMode(),
+                Matchers.anyOf(equalTo(OperationMode.STOPPING),
+                    equalTo(OperationMode.STOPPED)));
+
+        StartILMRequest startReq = new StartILMRequest();
+        AcknowledgedResponse startResponse = execute(startReq, highLevelClient().indexLifecycle()::startILM,
+                highLevelClient().indexLifecycle()::startILMAsync);
+        assertTrue(startResponse.isAcknowledged());
+
+        statusResponse = execute(statusRequest, highLevelClient().indexLifecycle()::lifecycleManagementStatus,
+            highLevelClient().indexLifecycle()::lifecycleManagementStatusAsync);
+        assertEquals(statusResponse.getOperationMode(), OperationMode.RUNNING);
+    }
+
+    public void testExplainLifecycle() throws Exception {
+        Map<String, Phase> lifecyclePhases = new HashMap<>();
+        Map<String, LifecycleAction> hotActions = Collections.singletonMap(
+            RolloverAction.NAME,
+            new RolloverAction(null, TimeValue.timeValueHours(50 * 24), null));
+        Phase hotPhase = new Phase("hot", randomFrom(TimeValue.ZERO, null), hotActions);
+        lifecyclePhases.put("hot", hotPhase);
+
+        Map<String, LifecycleAction> warmActions = new HashMap<>();
+        warmActions.put(AllocateAction.NAME, new AllocateAction(null, null, null, Collections.singletonMap("_name", "node-1")));
+        warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
+        warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1000));
+        lifecyclePhases.put("warm", new Phase("warm", TimeValue.timeValueSeconds(1000), warmActions));
+
+        Map<String, LifecycleAction> coldActions = new HashMap<>();
+        coldActions.put(AllocateAction.NAME, new AllocateAction(0, null, null, null));
+        lifecyclePhases.put("cold", new Phase("cold", TimeValue.timeValueSeconds(2000), coldActions));
+
+        Map<String, LifecycleAction> deleteActions = Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
+        lifecyclePhases.put("delete", new Phase("delete", TimeValue.timeValueSeconds(3000), deleteActions));
+
+        LifecyclePolicy policy = new LifecyclePolicy(randomAlphaOfLength(10), lifecyclePhases);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+        AcknowledgedResponse putResponse = execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync);
+        assertTrue(putResponse.isAcknowledged());
+        GetLifecyclePolicyRequest getRequest = new GetLifecyclePolicyRequest(policy.getName());
+        GetLifecyclePolicyResponse getResponse = execute(getRequest, highLevelClient().indexLifecycle()::getLifecyclePolicy,
+            highLevelClient().indexLifecycle()::getLifecyclePolicyAsync);
+        long expectedPolicyModifiedDate = getResponse.getPolicies().get(policy.getName()).getModifiedDate();
+
+
+        createIndex("foo-01", Settings.builder().put("index.lifecycle.name", policy.getName())
+            .put("index.lifecycle.rollover_alias", "foo-alias").build(), "", "\"foo-alias\" : {}");
+
+        createIndex("baz-01", Settings.builder().put("index.lifecycle.name", policy.getName())
+            .put("index.lifecycle.rollover_alias", "baz-alias").build(), "", "\"baz-alias\" : {}");
+
+        createIndex("squash", Settings.EMPTY);
+
+        ExplainLifecycleRequest req = new ExplainLifecycleRequest();
+        req.indices("foo-01", "baz-01", "squash");
+        ExplainLifecycleResponse response = execute(req, highLevelClient().indexLifecycle()::explainLifecycle,
+                highLevelClient().indexLifecycle()::explainLifecycleAsync);
+        Map<String, IndexLifecycleExplainResponse> indexResponses = response.getIndexResponses();
+        assertEquals(3, indexResponses.size());
+        IndexLifecycleExplainResponse fooResponse = indexResponses.get("foo-01");
+        assertNotNull(fooResponse);
+        assertTrue(fooResponse.managedByILM());
+        assertEquals("foo-01", fooResponse.getIndex());
+        assertEquals("hot", fooResponse.getPhase());
+        assertEquals("rollover", fooResponse.getAction());
+        assertEquals("attempt_rollover", fooResponse.getStep());
+        assertEquals(new PhaseExecutionInfo(policy.getName(), new Phase("", hotPhase.getMinimumAge(), hotPhase.getActions()),
+                1L, expectedPolicyModifiedDate), fooResponse.getPhaseExecutionInfo());
+        IndexLifecycleExplainResponse bazResponse = indexResponses.get("baz-01");
+        assertNotNull(bazResponse);
+        assertTrue(bazResponse.managedByILM());
+        assertEquals("baz-01", bazResponse.getIndex());
+        assertEquals("hot", bazResponse.getPhase());
+        assertEquals("rollover", bazResponse.getAction());
+        assertEquals("attempt_rollover", bazResponse.getStep());
+        IndexLifecycleExplainResponse squashResponse = indexResponses.get("squash");
+        assertNotNull(squashResponse);
+        assertFalse(squashResponse.managedByILM());
+        assertEquals("squash", squashResponse.getIndex());
+    }
+
+    public void testDeleteLifecycle() throws IOException {
+        String policyName = randomAlphaOfLength(10);
+        LifecyclePolicy policy = createRandomPolicy(policyName);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+        assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+
+        DeleteLifecyclePolicyRequest deleteRequest = new DeleteLifecyclePolicyRequest(policy.getName());
+        assertAcked(execute(deleteRequest, highLevelClient().indexLifecycle()::deleteLifecyclePolicy,
+            highLevelClient().indexLifecycle()::deleteLifecyclePolicyAsync));
+
+        GetLifecyclePolicyRequest getRequest = new GetLifecyclePolicyRequest(policyName);
+        ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class,
+            () -> execute(getRequest, highLevelClient().indexLifecycle()::getLifecyclePolicy,
+                highLevelClient().indexLifecycle()::getLifecyclePolicyAsync));
+        assertEquals(404, ex.status().getStatus());
+    }
+
+    public void testPutLifecycle() throws IOException {
+        String name = randomAlphaOfLengthBetween(5, 20);
+        LifecyclePolicy policy = createRandomPolicy(name);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+
+        assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+
+        GetLifecyclePolicyRequest getRequest = new GetLifecyclePolicyRequest(name);
+        GetLifecyclePolicyResponse response = execute(getRequest, highLevelClient().indexLifecycle()::getLifecyclePolicy,
+            highLevelClient().indexLifecycle()::getLifecyclePolicyAsync);
+        assertEquals(policy, response.getPolicies().get(name).getPolicy());
+    }
+
+    public void testGetMultipleLifecyclePolicies() throws IOException {
+        int numPolicies = randomIntBetween(1, 10);
+        String[] policyNames = new String[numPolicies];
+        LifecyclePolicy[] policies = new LifecyclePolicy[numPolicies];
+        for (int i = 0; i < numPolicies; i++) {
+            policyNames[i] = "policy-" + randomAlphaOfLengthBetween(5, 10);
+            policies[i] = createRandomPolicy(policyNames[i]);
+            PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policies[i]);
+            assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+                highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+        }
+
+        GetLifecyclePolicyRequest getRequest = new GetLifecyclePolicyRequest(randomFrom(policyNames, null));
+        GetLifecyclePolicyResponse response = execute(getRequest, highLevelClient().indexLifecycle()::getLifecyclePolicy,
+            highLevelClient().indexLifecycle()::getLifecyclePolicyAsync);
+        List<LifecyclePolicy> retrievedPolicies = Arrays.stream(response.getPolicies().values().toArray())
+            .map(p -> ((LifecyclePolicyMetadata) p).getPolicy()).collect(Collectors.toList());
+        assertThat(retrievedPolicies, hasItems(policies));
+    }
+
+    public void testRetryLifecycleStep() throws IOException {
+        String policyName = randomAlphaOfLength(10);
+        LifecyclePolicy policy = createRandomPolicy(policyName);
+        PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
+        assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+        createIndex("retry", Settings.builder().put("index.lifecycle.name", policy.getName()).build());
+        RetryLifecyclePolicyRequest retryRequest = new RetryLifecyclePolicyRequest("retry");
+        ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class,
+            () -> execute(
+                retryRequest, highLevelClient().indexLifecycle()::retryLifecycleStep,
+                highLevelClient().indexLifecycle()::retryLifecycleStepAsync
+            )
+        );
+        assertEquals(400, ex.status().getStatus());
+        assertEquals(
+            "Elasticsearch exception [type=illegal_argument_exception, reason=cannot retry an action for an index [retry]" +
+                " that has not encountered an error when running a Lifecycle Policy]",
+            ex.getRootCause().getMessage()
+        );
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleRequestConvertersTests.java
new file mode 100644
index 0000000000000..1af29701bc7c8
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleRequestConvertersTests.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
+import org.elasticsearch.client.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RemoveIndexLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.RetryLifecyclePolicyRequest;
+import org.elasticsearch.client.indexlifecycle.StartILMRequest;
+import org.elasticsearch.client.indexlifecycle.StopILMRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.client.RequestConvertersTests.randomIndicesNames;
+import static org.elasticsearch.client.RequestConvertersTests.setRandomIndicesOptions;
+import static org.elasticsearch.client.RequestConvertersTests.setRandomMasterTimeout;
+import static org.elasticsearch.client.RequestConvertersTests.setRandomTimeoutTimeValue;
+import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class IndexLifecycleRequestConvertersTests extends ESTestCase {
+
+    public void testGetLifecyclePolicy() {
+        String[] policies = rarely() ? null : randomIndicesNames(0, 10);
+        GetLifecyclePolicyRequest req = new GetLifecyclePolicyRequest(policies);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.getLifecyclePolicy(req);
+        assertEquals(request.getMethod(), HttpGet.METHOD_NAME);
+        String policiesStr = Strings.arrayToCommaDelimitedString(policies);
+        assertEquals(request.getEndpoint(), "/_ilm/policy" + (policiesStr.isEmpty() ? "" : ("/" + policiesStr)));
+        assertEquals(request.getParameters(), expectedParams);
+    }
+
+    public void testPutLifecyclePolicy() throws Exception {
+        String name = randomAlphaOfLengthBetween(2, 20);
+        LifecyclePolicy policy = createRandomPolicy(name);
+        PutLifecyclePolicyRequest req = new PutLifecyclePolicyRequest(policy);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.putLifecyclePolicy(req);
+        assertEquals(HttpPut.METHOD_NAME, request.getMethod());
+        assertEquals("/_ilm/policy/" + name, request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+    }
+
+    public void testDeleteLifecycle() {
+        String lifecycleName = randomAlphaOfLengthBetween(2,20);
+        DeleteLifecyclePolicyRequest req = new DeleteLifecyclePolicyRequest(lifecycleName);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.deleteLifecyclePolicy(req);
+        assertEquals(request.getMethod(), HttpDelete.METHOD_NAME);
+        assertEquals(request.getEndpoint(), "/_ilm/policy/" + lifecycleName);
+        assertEquals(request.getParameters(), expectedParams);
+    }
+
+    public void testRemoveIndexLifecyclePolicy() {
+        Map<String, String> expectedParams = new HashMap<>();
+        String[] indices = randomIndicesNames(0, 10);
+        IndicesOptions indicesOptions = setRandomIndicesOptions(IndicesOptions.strictExpandOpen(), expectedParams);
+        RemoveIndexLifecyclePolicyRequest req = new RemoveIndexLifecyclePolicyRequest(Arrays.asList(indices), indicesOptions);
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.removeIndexLifecyclePolicy(req);
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        String idxString = Strings.arrayToCommaDelimitedString(indices);
+        assertThat(request.getEndpoint(), equalTo("/" + (idxString.isEmpty() ? "" : (idxString + "/")) + "_ilm/remove"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+
+    public void testStartILM() throws Exception {
+        StartILMRequest req = new StartILMRequest();
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.startILM(req);
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        assertThat(request.getEndpoint(), equalTo("/_ilm/start"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+
+    public void testStopILM() throws Exception {
+        StopILMRequest req = new StopILMRequest();
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.stopILM(req);
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        assertThat(request.getEndpoint(), equalTo("/_ilm/stop"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+
+    public void testLifecycleManagementStatus() throws Exception {
+        LifecycleManagementStatusRequest req = new LifecycleManagementStatusRequest();
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.lifecycleManagementStatus(req);
+        assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME));
+        assertThat(request.getEndpoint(), equalTo("/_ilm/status"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+
+    public void testExplainLifecycle() throws Exception {
+        ExplainLifecycleRequest req = new ExplainLifecycleRequest();
+        String[] indices = rarely() ? null : randomIndicesNames(0, 10);
+        req.indices(indices);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req, expectedParams);
+        setRandomIndicesOptions(req::indicesOptions, req::indicesOptions, expectedParams);
+
+        Request request = IndexLifecycleRequestConverters.explainLifecycle(req);
+        assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME));
+        String idxString = Strings.arrayToCommaDelimitedString(indices);
+        assertThat(request.getEndpoint(), equalTo("/" + (idxString.isEmpty() ? "" : (idxString + "/")) + "_ilm/explain"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+
+    public void testRetryLifecycle() throws Exception {
+        String[] indices = randomIndicesNames(1, 10);
+        RetryLifecyclePolicyRequest req = new RetryLifecyclePolicyRequest(indices);
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomMasterTimeout(req::setMasterTimeout, TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT, expectedParams);
+        setRandomTimeoutTimeValue(req::setTimeout, TimedRequest.DEFAULT_ACK_TIMEOUT, expectedParams);
+        Request request = IndexLifecycleRequestConverters.retryLifecycle(req);
+        assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        String idxString = Strings.arrayToCommaDelimitedString(indices);
+        assertThat(request.getEndpoint(), equalTo("/" + (idxString.isEmpty() ? "" : (idxString + "/")) + "_ilm/retry"));
+        assertThat(request.getParameters(), equalTo(expectedParams));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
index 2c421597c68d0..927183ba7715f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -534,6 +534,9 @@ public void testUpdateAliases() throws IOException {
 
         IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
         AliasActions addAction = new AliasActions(AliasActions.Type.ADD).index(index).aliases(alias);
+        if (randomBoolean()) {
+            addAction.writeIndex(randomBoolean());
+        }
         addAction.routing("routing").searchRouting("search_routing").filter("{\"term\":{\"year\":2016}}");
         aliasesAddRequest.addAliasAction(addAction);
         AcknowledgedResponse aliasesAddResponse = execute(aliasesAddRequest, highLevelClient().indices()::updateAliases,
@@ -548,6 +551,8 @@ public void testUpdateAliases() throws IOException {
         Map<String, Object> filter = (Map<String, Object>) getAlias.get("filter");
         Map<String, Object> term = (Map<String, Object>) filter.get("term");
         assertEquals(2016, term.get("year"));
+        Boolean isWriteIndex = (Boolean) getAlias.get("is_write_index");
+        assertThat(isWriteIndex, equalTo(addAction.writeIndex()));
 
         String alias2 = "alias2";
         IndicesAliasesRequest aliasesAddRemoveRequest = new IndicesAliasesRequest();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
index d4fdfb2c995df..bb0dbf6680127 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.client.ml.PreviewDatafeedRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
+import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequestTests;
@@ -59,6 +60,8 @@
 import org.elasticsearch.client.ml.job.config.Job;
 import org.elasticsearch.client.ml.job.config.JobUpdate;
 import org.elasticsearch.client.ml.job.config.JobUpdateTests;
+import org.elasticsearch.client.ml.job.config.MlFilter;
+import org.elasticsearch.client.ml.job.config.MlFilterTests;
 import org.elasticsearch.client.ml.job.util.PageParams;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -511,6 +514,20 @@ public void testDeleteCalendar() {
         assertEquals("/_xpack/ml/calendars/" + deleteCalendarRequest.getCalendarId(), request.getEndpoint());
     }
 
+    public void testPutFilter() throws IOException {
+        MlFilter filter = MlFilterTests.createRandom("foo");
+        PutFilterRequest putFilterRequest = new PutFilterRequest(filter);
+
+        Request request = MLRequestConverters.putFilter(putFilterRequest);
+
+        assertEquals(HttpPut.METHOD_NAME, request.getMethod());
+        assertThat(request.getEndpoint(), equalTo("/_xpack/ml/filters/foo"));
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            MlFilter parsedFilter = MlFilter.PARSER.apply(parser, null).build();
+            assertThat(parsedFilter, equalTo(filter));
+        }
+    }
+
     private static Job createValidJob(String jobId) {
         AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
                 Detector.builder().setFunction("count").build()));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 951e6209d6b8c..ff3218795e435 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -58,6 +58,8 @@
 import org.elasticsearch.client.ml.PutCalendarResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
+import org.elasticsearch.client.ml.PutFilterRequest;
+import org.elasticsearch.client.ml.PutFilterResponse;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
@@ -78,6 +80,7 @@
 import org.elasticsearch.client.ml.job.config.Job;
 import org.elasticsearch.client.ml.job.config.JobState;
 import org.elasticsearch.client.ml.job.config.JobUpdate;
+import org.elasticsearch.client.ml.job.config.MlFilter;
 import org.elasticsearch.client.ml.job.stats.JobStats;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -859,6 +862,22 @@ public void testDeleteCalendar() throws IOException {
         assertThat(exception.status().getStatus(), equalTo(404));
     }
 
+    public void testFilterJob() throws Exception {
+        String filterId = "filter-job-test";
+        MlFilter mlFilter = MlFilter.builder(filterId)
+            .setDescription(randomAlphaOfLength(10))
+            .setItems(generateRandomStringArray(10, 10, false, false))
+            .build();
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        PutFilterResponse putFilterResponse = execute(new PutFilterRequest(mlFilter),
+            machineLearningClient::putFilter,
+            machineLearningClient::putFilterAsync);
+        MlFilter createdFilter = putFilterResponse.getResponse();
+
+        assertThat(createdFilter, equalTo(mlFilter));
+    }
+
     public static String randomValidJobId() {
         CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
         return generator.ofCodePointsLength(random(), 10, 10);
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
index e6fdeffa142e9..06ae5e2090c65 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java
@@ -109,9 +109,8 @@ public void testRankEvalRequest() throws IOException {
 
         // now try this when test2 is closed
         client().performRequest(new Request("POST", "index2/_close"));
-        rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
-        response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync,
-                highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
+        rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
     }
 
     private static List<RatedRequest> createTestEvaluationSpec() {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 32b07316f6a9e..84aabb97ae798 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -56,6 +56,7 @@
 import org.elasticsearch.client.core.TermVectorsRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.RequestConverters.EndpointBuilder;
+import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -980,6 +981,72 @@ public void testSearchNullIndicesAndTypes() {
         expectThrows(NullPointerException.class, () -> new SearchRequest().types((String[]) null));
     }
 
+     public void testCountNotNullSource() throws IOException {
+        //as we create SearchSourceBuilder in CountRequest constructor
+        CountRequest countRequest = new CountRequest();
+        Request request = RequestConverters.count(countRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_count", request.getEndpoint());
+        assertNotNull(request.getEntity());
+    }
+
+    public void testCount() throws Exception {
+        String[] indices = randomIndicesNames(0, 5);
+        CountRequest countRequest = new CountRequest(indices);
+
+        int numTypes = randomIntBetween(0, 5);
+        String[] types = new String[numTypes];
+        for (int i = 0; i < numTypes; i++) {
+            types[i] = "type-" + randomAlphaOfLengthBetween(2, 5);
+        }
+        countRequest.types(types);
+
+        Map<String, String> expectedParams = new HashMap<>();
+        setRandomCountParams(countRequest, expectedParams);
+        setRandomIndicesOptions(countRequest::indicesOptions, countRequest::indicesOptions, expectedParams);
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        if (frequently()) {
+            if (randomBoolean()) {
+                searchSourceBuilder.minScore(randomFloat());
+            }
+        }
+        countRequest.source(searchSourceBuilder);
+        Request request = RequestConverters.count(countRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String index = String.join(",", indices);
+        if (Strings.hasLength(index)) {
+            endpoint.add(index);
+        }
+        String type = String.join(",", types);
+        if (Strings.hasLength(type)) {
+            endpoint.add(type);
+        }
+        endpoint.add("_count");
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertToXContentBody(searchSourceBuilder, request.getEntity());
+    }
+
+    public void testCountNullIndicesAndTypes() {
+        expectThrows(NullPointerException.class, () -> new CountRequest((String[]) null));
+        expectThrows(NullPointerException.class, () -> new CountRequest().indices((String[]) null));
+        expectThrows(NullPointerException.class, () -> new CountRequest().types((String[]) null));
+    }
+
+    private static void setRandomCountParams(CountRequest countRequest,
+                                             Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            countRequest.routing(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("routing", countRequest.routing());
+        }
+        if (randomBoolean()) {
+            countRequest.preference(randomAlphaOfLengthBetween(3, 10));
+            expectedParams.put("preference", countRequest.preference());
+        }
+    }
+
     public void testMultiSearch() throws IOException {
         int numberOfSearchRequests = randomIntBetween(0, 32);
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
@@ -1000,7 +1067,8 @@ public void testMultiSearch() throws IOException {
             IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions();
             searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(),
                     randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(),
-                    msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()));
+                    msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(),
+                msearchDefault.ignoreThrottled()));
             multiSearchRequest.add(searchRequest);
         }
 
@@ -1610,6 +1678,24 @@ static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<In
         }
     }
 
+    static IndicesOptions setRandomIndicesOptions(IndicesOptions indicesOptions, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+        }
+        expectedParams.put("ignore_unavailable", Boolean.toString(indicesOptions.ignoreUnavailable()));
+        expectedParams.put("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
+        if (indicesOptions.expandWildcardsOpen() && indicesOptions.expandWildcardsClosed()) {
+            expectedParams.put("expand_wildcards", "open,closed");
+        } else if (indicesOptions.expandWildcardsOpen()) {
+            expectedParams.put("expand_wildcards", "open");
+        } else if (indicesOptions.expandWildcardsClosed()) {
+            expectedParams.put("expand_wildcards", "closed");
+        } else {
+            expectedParams.put("expand_wildcards", "none");
+        }
+        return indicesOptions;
+    }
+
     static void setRandomIncludeDefaults(GetIndexRequest request, Map<String, String> expectedParams) {
         if (randomBoolean()) {
             boolean includeDefaults = randomBoolean();
@@ -1660,6 +1746,17 @@ static void setRandomTimeout(Consumer<String> setter, TimeValue defaultTimeout,
         }
     }
 
+    static void setRandomTimeoutTimeValue(Consumer<TimeValue> setter, TimeValue defaultTimeout,
+                                                  Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            TimeValue timeout = TimeValue.parseTimeValue(randomTimeValue(), "random_timeout");
+            setter.accept(timeout);
+            expectedParams.put("timeout", timeout.getStringRep());
+        } else {
+            expectedParams.put("timeout", defaultTimeout.getStringRep());
+        }
+    }
+
     static void setRandomMasterTimeout(MasterNodeRequest<?> request, Map<String, String> expectedParams) {
         setRandomMasterTimeout(request::masterNodeTimeout, expectedParams);
     }
@@ -1680,6 +1777,16 @@ static void setRandomMasterTimeout(Consumer<String> setter, Map<String, String>
         }
     }
 
+    static void setRandomMasterTimeout(Consumer<TimeValue> setter, TimeValue defaultTimeout, Map<String, String> expectedParams) {
+        if (randomBoolean()) {
+            TimeValue masterTimeout = TimeValue.parseTimeValue(randomTimeValue(), "random_master_timeout");
+            setter.accept(masterTimeout);
+            expectedParams.put("master_timeout", masterTimeout.getStringRep());
+        } else {
+            expectedParams.put("master_timeout", defaultTimeout.getStringRep());
+        }
+    }
+
     static void setRandomWaitForActiveShards(Consumer<ActiveShardCount> setter, Map<String, String> expectedParams) {
         setRandomWaitForActiveShards(setter, ActiveShardCount.DEFAULT, expectedParams);
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index b66a96f786597..4adec512b5d8f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -50,6 +49,13 @@
 import org.elasticsearch.action.search.SearchResponseSections;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.indexlifecycle.AllocateAction;
+import org.elasticsearch.client.indexlifecycle.DeleteAction;
+import org.elasticsearch.client.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.client.indexlifecycle.LifecycleAction;
+import org.elasticsearch.client.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.client.indexlifecycle.RolloverAction;
+import org.elasticsearch.client.indexlifecycle.ShrinkAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -619,7 +625,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(10, namedXContents.size());
+        assertEquals(16, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -629,7 +635,7 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals(3, categories.size());
+        assertEquals(4, categories.size());
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
@@ -643,6 +649,13 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(MeanReciprocalRank.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(ExpectedReciprocalRank.NAME));
+        assertEquals(Integer.valueOf(6), categories.get(LifecycleAction.class));
+        assertTrue(names.contains(AllocateAction.NAME));
+        assertTrue(names.contains(DeleteAction.NAME));
+        assertTrue(names.contains(ForceMergeAction.NAME));
+        assertTrue(names.contains(ReadOnlyAction.NAME));
+        assertTrue(names.contains(RolloverAction.NAME));
+        assertTrue(names.contains(ShrinkAction.NAME));
     }
 
     public void testMethodWithHeadersArgumentAreDeprecated() {
@@ -664,7 +677,6 @@ public void testApiNamingConventions() throws Exception {
         //this list should be empty once the high-level client is feature complete
         String[] notYetSupportedApi = new String[]{
             "cluster.remote_info",
-            "count",
             "create",
             "get_source",
             "indices.delete_alias",
@@ -737,7 +749,7 @@ public void testApiNamingConventions() throws Exception {
                         methods.containsKey(apiName.substring(0, apiName.length() - 6)));
                 assertThat("async method [" + method + "] should return void", method.getReturnType(), equalTo(Void.TYPE));
                 assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length);
-                if (apiName.equals("security.get_ssl_certificates_async")) {
+                if (apiName.equals("security.authenticate_async") || apiName.equals("security.get_ssl_certificates_async")) {
                     assertEquals(2, method.getParameterTypes().length);
                     assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class));
                     assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class));
@@ -762,7 +774,8 @@ public void testApiNamingConventions() throws Exception {
 
                 assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length);
                 //a few methods don't accept a request object as argument
-                if (apiName.equals("ping") || apiName.equals("info") || apiName.equals("security.get_ssl_certificates")) {
+                if (apiName.equals("ping") || apiName.equals("info") || apiName.equals("security.get_ssl_certificates")
+                    || apiName.equals("security.authenticate")) {
                     assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length);
                     assertThat("the parameter to method [" + method + "] is the wrong type",
                         method.getParameterTypes()[0], equalTo(RequestOptions.class));
@@ -788,7 +801,8 @@ public void testApiNamingConventions() throws Exception {
                             apiName.startsWith("watcher.") == false &&
                             apiName.startsWith("graph.") == false &&
                             apiName.startsWith("migration.") == false &&
-                            apiName.startsWith("security.") == false) {
+                            apiName.startsWith("security.") == false &&
+                            apiName.startsWith("index_lifecycle.") == false) {
                             apiNotFound.add(apiName);
                         }
                     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
index ed992a95fe1bd..6b790659a6f8a 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -34,6 +35,8 @@
 import org.elasticsearch.client.rollup.GetRollupJobResponse;
 import org.elasticsearch.client.rollup.GetRollupJobResponse.IndexerState;
 import org.elasticsearch.client.rollup.GetRollupJobResponse.JobWrapper;
+import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
+import org.elasticsearch.client.rollup.DeleteRollupJobResponse;
 import org.elasticsearch.client.rollup.PutRollupJobRequest;
 import org.elasticsearch.client.rollup.PutRollupJobResponse;
 import org.elasticsearch.client.rollup.RollableIndexCaps;
@@ -53,6 +56,7 @@
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.junit.Before;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -69,18 +73,35 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 
 public class RollupIT extends ESRestHighLevelClientTestCase {
 
+    double sum = 0.0d;
+    int max = Integer.MIN_VALUE;
+    int min = Integer.MAX_VALUE;
     private static final List<String> SUPPORTED_METRICS = Arrays.asList(MaxAggregationBuilder.NAME, MinAggregationBuilder.NAME,
-            SumAggregationBuilder.NAME, AvgAggregationBuilder.NAME, ValueCountAggregationBuilder.NAME);
-
-    public void testPutStartAndGetRollupJob() throws Exception {
-        double sum = 0.0d;
-        int max = Integer.MIN_VALUE;
-        int min = Integer.MAX_VALUE;
+        SumAggregationBuilder.NAME, AvgAggregationBuilder.NAME, ValueCountAggregationBuilder.NAME);
+
+    private String id;
+    private String indexPattern;
+    private String rollupIndex;
+    private String cron;
+    private int pageSize;
+    private int numDocs;
+
+    @Before
+    public void init() throws Exception {
+        id = randomAlphaOfLength(10);
+        indexPattern = randomFrom("docs", "d*", "doc*");
+        rollupIndex = randomFrom("rollup", "test");
+        cron = "*/1 * * * * ?";
+        numDocs = indexDocs();
+        pageSize = randomIntBetween(numDocs, numDocs * 10);
+    }
 
+    public int indexDocs() throws Exception {
         final BulkRequest bulkRequest = new BulkRequest();
         bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         for (int minute = 0; minute < 60; minute++) {
@@ -120,12 +141,33 @@ public void testPutStartAndGetRollupJob() throws Exception {
 
         RefreshResponse refreshResponse = highLevelClient().indices().refresh(new RefreshRequest("docs"), RequestOptions.DEFAULT);
         assertEquals(0, refreshResponse.getFailedShards());
+        return numDocs;
+    }
 
-        final String id = randomAlphaOfLength(10);
-        final String indexPattern = randomFrom("docs", "d*", "doc*");
-        final String rollupIndex = randomFrom("rollup", "test");
-        final String cron = "*/1 * * * * ?";
-        final int pageSize = randomIntBetween(numDocs, numDocs * 10);
+
+    public void testDeleteRollupJob() throws Exception {
+        final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
+        final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
+        final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(30, 600));
+        PutRollupJobRequest putRollupJobRequest =
+            new PutRollupJobRequest(new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groups, metrics, timeout));
+        final RollupClient rollupClient = highLevelClient().rollup();
+        PutRollupJobResponse response = execute(putRollupJobRequest, rollupClient::putRollupJob, rollupClient::putRollupJobAsync);
+        DeleteRollupJobRequest deleteRollupJobRequest = new DeleteRollupJobRequest(id);
+        DeleteRollupJobResponse deleteRollupJobResponse = highLevelClient().rollup()
+            .deleteRollupJob(deleteRollupJobRequest, RequestOptions.DEFAULT);
+        assertTrue(deleteRollupJobResponse.isAcknowledged());
+    }
+
+    public void testDeleteMissingRollupJob() {
+        DeleteRollupJobRequest deleteRollupJobRequest = new DeleteRollupJobRequest(randomAlphaOfLength(10));
+        ElasticsearchStatusException responseException = expectThrows(ElasticsearchStatusException.class,() -> highLevelClient().rollup()
+            .deleteRollupJob(deleteRollupJobRequest, RequestOptions.DEFAULT));
+        assertThat(responseException.status().getStatus(), is(404));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testPutAndGetRollupJob() throws Exception {
         // TODO expand this to also test with histogram and terms?
         final GroupConfig groups = new GroupConfig(new DateHistogramGroupConfig("date", DateHistogramInterval.DAY));
         final List<MetricConfig> metrics = Collections.singletonList(new MetricConfig("value", SUPPORTED_METRICS));
@@ -142,9 +184,6 @@ public void testPutStartAndGetRollupJob() throws Exception {
         StartRollupJobResponse startResponse = execute(startRequest, rollupClient::startRollupJob, rollupClient::startRollupJobAsync);
         assertTrue(startResponse.isAcknowledged());
 
-        int finalMin = min;
-        int finalMax = max;
-        double finalSum = sum;
         assertBusy(() -> {
             SearchResponse searchResponse = highLevelClient().search(new SearchRequest(rollupIndex), RequestOptions.DEFAULT);
             assertEquals(0, searchResponse.getFailedShards());
@@ -162,13 +201,13 @@ public void testPutStartAndGetRollupJob() throws Exception {
                 for (String name : metric.getMetrics()) {
                     Number value = (Number) source.get(metric.getField() + "." + name + ".value");
                     if ("min".equals(name)) {
-                        assertEquals(finalMin, value.intValue());
+                        assertEquals(min, value.intValue());
                     } else if ("max".equals(name)) {
-                        assertEquals(finalMax, value.intValue());
+                        assertEquals(max, value.intValue());
                     } else if ("sum".equals(name)) {
-                        assertEquals(finalSum, value.doubleValue(), 0.0d);
+                        assertEquals(sum, value.doubleValue(), 0.0d);
                     } else if ("avg".equals(name)) {
-                        assertEquals(finalSum, value.doubleValue(), 0.0d);
+                        assertEquals(sum, value.doubleValue(), 0.0d);
                         Number avgCount = (Number) source.get(metric.getField() + "." + name + "._count");
                         assertEquals(numDocs, avgCount.intValue());
                     } else if ("value_count".equals(name)) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
index 2458cb9c1f20b..8468636d5cdb8 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -35,6 +35,8 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
@@ -1249,4 +1251,69 @@ private static void assertSearchHeader(SearchResponse searchResponse) {
         assertEquals(0, searchResponse.getShardFailures().length);
         assertEquals(SearchResponse.Clusters.EMPTY, searchResponse.getClusters());
     }
+
+    public void testCountOneIndexNoQuery() throws IOException {
+        CountRequest countRequest = new CountRequest("index");
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(5, countResponse.getCount());
+    }
+
+    public void testCountMultipleIndicesNoQuery() throws IOException {
+        CountRequest countRequest = new CountRequest("index", "index1");
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(7, countResponse.getCount());
+    }
+
+    public void testCountAllIndicesNoQuery() throws IOException {
+        CountRequest countRequest = new CountRequest();
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(12, countResponse.getCount());
+    }
+
+    public void testCountOneIndexMatchQuery() throws IOException {
+        CountRequest countRequest = new CountRequest("index");
+        countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10)));
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(1, countResponse.getCount());
+    }
+
+    public void testCountMultipleIndicesMatchQueryUsingConstructor() throws IOException {
+
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1"));
+        CountRequest countRequest = new CountRequest(new String[]{"index1", "index2", "index3"}, sourceBuilder);
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(3, countResponse.getCount());
+
+    }
+
+    public void testCountMultipleIndicesMatchQuery() throws IOException {
+
+        CountRequest countRequest = new CountRequest("index1", "index2", "index3");
+        countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1")));
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(3, countResponse.getCount());
+    }
+
+    public void testCountAllIndicesMatchQuery() throws IOException {
+
+        CountRequest countRequest = new CountRequest();
+        countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1")));
+        CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
+        assertCountHeader(countResponse);
+        assertEquals(3, countResponse.getCount());
+    }
+
+    private static void assertCountHeader(CountResponse countResponse) {
+        assertEquals(0, countResponse.getSkippedShards());
+        assertEquals(0, countResponse.getFailedShards());
+        assertThat(countResponse.getTotalShards(), greaterThan(0));
+        assertEquals(countResponse.getTotalShards(), countResponse.getSuccessfulShards());
+        assertEquals(0, countResponse.getShardFailures().length);
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java
new file mode 100644
index 0000000000000..74a4d58e2bf77
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.client.security.AuthenticateResponse;
+import org.elasticsearch.client.security.PutUserRequest;
+import org.elasticsearch.client.security.PutUserResponse;
+import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.common.CharArrays;
+
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+
+public class SecurityIT extends ESRestHighLevelClientTestCase {
+
+    public void testAuthenticate() throws Exception {
+        final SecurityClient securityClient = highLevelClient().security();
+        // test fixture: put enabled user
+        final PutUserRequest putUserRequest = randomPutUserRequest(true);
+        final PutUserResponse putUserResponse = execute(putUserRequest, securityClient::putUser, securityClient::putUserAsync);
+        assertThat(putUserResponse.isCreated(), is(true));
+
+        // authenticate correctly
+        final String basicAuthHeader = basicAuthHeader(putUserRequest.getUsername(), putUserRequest.getPassword());
+        final AuthenticateResponse authenticateResponse = execute(securityClient::authenticate, securityClient::authenticateAsync,
+                authorizationRequestOptions(basicAuthHeader));
+
+        assertThat(authenticateResponse.getUser().username(), is(putUserRequest.getUsername()));
+        if (putUserRequest.getRoles().isEmpty()) {
+            assertThat(authenticateResponse.getUser().roles(), is(empty()));
+        } else {
+            assertThat(authenticateResponse.getUser().roles(), contains(putUserRequest.getRoles().toArray()));
+        }
+        assertThat(authenticateResponse.getUser().metadata(), is(putUserRequest.getMetadata()));
+        assertThat(authenticateResponse.getUser().fullName(), is(putUserRequest.getFullName()));
+        assertThat(authenticateResponse.getUser().email(), is(putUserRequest.getEmail()));
+        assertThat(authenticateResponse.enabled(), is(true));
+
+        // delete user
+        final Request deleteUserRequest = new Request(HttpDelete.METHOD_NAME, "/_xpack/security/user/" + putUserRequest.getUsername());
+        highLevelClient().getLowLevelClient().performRequest(deleteUserRequest);
+
+        // authentication no longer works
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> execute(securityClient::authenticate,
+                securityClient::authenticateAsync, authorizationRequestOptions(basicAuthHeader)));
+        assertThat(e.getMessage(), containsString("unable to authenticate user [" + putUserRequest.getUsername() + "]"));
+    }
+
+    private static PutUserRequest randomPutUserRequest(boolean enabled) {
+        final String username = randomAlphaOfLengthBetween(1, 4);
+        final char[] password = randomAlphaOfLengthBetween(6, 10).toCharArray();
+        final List<String> roles = Arrays.asList(generateRandomStringArray(3, 3, false, true));
+        final String fullName = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 3));
+        final String email = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 3));
+        final Map<String, Object> metadata;
+        metadata = new HashMap<>();
+        if (randomBoolean()) {
+            metadata.put("string", null);
+        } else {
+            metadata.put("string", randomAlphaOfLengthBetween(0, 4));
+        }
+        if (randomBoolean()) {
+            metadata.put("string_list", null);
+        } else {
+            metadata.put("string_list", Arrays.asList(generateRandomStringArray(4, 4, false, true)));
+        }
+        return new PutUserRequest(username, password, roles, fullName, email, enabled, metadata, RefreshPolicy.IMMEDIATE);
+    }
+    
+    private static String basicAuthHeader(String username, char[] password) {
+        final String concat = new StringBuilder().append(username).append(':').append(password).toString();
+        final byte[] concatBytes = CharArrays.toUtf8Bytes(concat.toCharArray());
+        return "Basic " + Base64.getEncoder().encodeToString(concatBytes);
+    }
+    
+    private static RequestOptions authorizationRequestOptions(String authorizationHeader) {
+        final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+        builder.addHeader("Authorization", authorizationHeader);
+        return builder.build();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TimedRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TimedRequestTests.java
new file mode 100644
index 0000000000000..8024aa0188598
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TimedRequestTests.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+public class TimedRequestTests extends ESTestCase {
+
+    public void testDefaults() {
+        TimedRequest timedRequest = new TimedRequest(){};
+        assertEquals(timedRequest.timeout(), TimedRequest.DEFAULT_ACK_TIMEOUT);
+        assertEquals(timedRequest.masterNodeTimeout(), TimedRequest.DEFAULT_MASTER_NODE_TIMEOUT);
+    }
+
+    public void testNonDefaults() {
+        TimedRequest timedRequest = new TimedRequest(){};
+        TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(0, 1000));
+        TimeValue masterTimeout = TimeValue.timeValueSeconds(randomIntBetween(0,1000));
+        timedRequest.setTimeout(timeout);
+        timedRequest.setMasterTimeout(masterTimeout);
+        assertEquals(timedRequest.timeout(), timeout);
+        assertEquals(timedRequest.masterNodeTimeout(), masterTimeout);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java
new file mode 100644
index 0000000000000..1030f4401e160
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.core;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+
+//similar to SearchRequestTests as CountRequest inline several members (and functionality) from SearchRequest
+public class CountRequestTests extends ESTestCase {
+
+    public void testIllegalArguments() {
+        CountRequest countRequest = new CountRequest();
+        assertNotNull(countRequest.indices());
+        assertNotNull(countRequest.indicesOptions());
+        assertNotNull(countRequest.types());
+
+        NullPointerException e = expectThrows(NullPointerException.class, () -> countRequest.indices((String[]) null));
+        assertEquals("indices must not be null", e.getMessage());
+        e = expectThrows(NullPointerException.class, () -> countRequest.indices((String) null));
+        assertEquals("index must not be null", e.getMessage());
+
+        e = expectThrows(NullPointerException.class, () -> countRequest.indicesOptions(null));
+        assertEquals("indicesOptions must not be null", e.getMessage());
+
+        e = expectThrows(NullPointerException.class, () -> countRequest.types((String[]) null));
+        assertEquals("types must not be null", e.getMessage());
+        e = expectThrows(NullPointerException.class, () -> countRequest.types((String) null));
+        assertEquals("type must not be null", e.getMessage());
+
+        e = expectThrows(NullPointerException.class, () -> countRequest.source(null));
+        assertEquals("source must not be null", e.getMessage());
+
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createCountRequest(), CountRequestTests::copyRequest, this::mutate);
+    }
+
+    private CountRequest createCountRequest() {
+        CountRequest countRequest = new CountRequest("index");
+        countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10)));
+        return countRequest;
+    }
+
+    private CountRequest mutate(CountRequest countRequest) {
+        CountRequest mutation = copyRequest(countRequest);
+        List<Runnable> mutators = new ArrayList<>();
+        mutators.add(() -> mutation.indices(ArrayUtils.concat(countRequest.indices(), new String[]{randomAlphaOfLength(10)})));
+        mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(countRequest.indicesOptions(),
+            () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))));
+        mutators.add(() -> mutation.types(ArrayUtils.concat(countRequest.types(), new String[]{randomAlphaOfLength(10)})));
+        mutators.add(() -> mutation.preference(randomValueOtherThan(countRequest.preference(), () -> randomAlphaOfLengthBetween(3, 10))));
+        mutators.add(() -> mutation.routing(randomValueOtherThan(countRequest.routing(), () -> randomAlphaOfLengthBetween(3, 10))));
+        randomFrom(mutators).run();
+        return mutation;
+    }
+
+    private static CountRequest copyRequest(CountRequest countRequest) {
+        CountRequest result = new CountRequest();
+        result.indices(countRequest.indices());
+        result.indicesOptions(countRequest.indicesOptions());
+        result.types(countRequest.types());
+        result.routing(countRequest.routing());
+        result.preference(countRequest.preference());
+        if (countRequest.source() != null) {
+            result.source(countRequest.source());
+        }
+        return result;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java
new file mode 100644
index 0000000000000..c2fc668d604e5
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.core;
+
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.action.RestActions;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public class CountResponseTests extends ESTestCase {
+
+    // Not comparing XContent for equivalence as we cannot compare the ShardSearchFailure#cause, because it will be wrapped in an outer
+    // ElasticSearchException. Best effort: try to check that the original message appears somewhere in the rendered xContent
+    // For more see ShardSearchFailureTests.
+    public void testFromXContent() throws IOException {
+        xContentTester(
+            this::createParser,
+            this::createTestInstance,
+            this::toXContent,
+            CountResponse::fromXContent)
+            .supportsUnknownFields(false)
+            .assertEqualsConsumer(this::assertEqualInstances)
+            .assertToXContentEquivalence(false)
+            .test();
+    }
+
+    private CountResponse createTestInstance() {
+        long count = 5;
+        Boolean terminatedEarly = randomBoolean() ? null : randomBoolean();
+        int totalShards = randomIntBetween(1, Integer.MAX_VALUE);
+        int successfulShards = randomIntBetween(0, totalShards);
+        int skippedShards = randomIntBetween(0, totalShards);
+        int numFailures = randomIntBetween(1, 5);
+        ShardSearchFailure[] failures = new ShardSearchFailure[numFailures];
+        for (int i = 0; i < failures.length; i++) {
+            failures[i] = createShardFailureTestItem();
+        }
+        CountResponse.ShardStats shardStats = new CountResponse.ShardStats(successfulShards, totalShards, skippedShards,
+            randomBoolean() ? ShardSearchFailure.EMPTY_ARRAY : failures);
+        return new CountResponse(count, terminatedEarly, shardStats);
+    }
+
+    private void toXContent(CountResponse response, XContentBuilder builder) throws IOException {
+        builder.startObject();
+        builder.field(CountResponse.COUNT.getPreferredName(), response.getCount());
+        if (response.isTerminatedEarly() != null) {
+            builder.field(CountResponse.TERMINATED_EARLY.getPreferredName(), response.isTerminatedEarly());
+        }
+        toXContent(response.getShardStats(), builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+    }
+
+    private void toXContent(CountResponse.ShardStats stats, XContentBuilder builder, ToXContent.Params params) throws IOException {
+        RestActions.buildBroadcastShardsHeader(builder, params, stats.getTotalShards(), stats.getSuccessfulShards(), stats
+            .getSkippedShards(), stats.getShardFailures().length, stats.getShardFailures());
+    }
+
+    @SuppressWarnings("Duplicates")
+    private static ShardSearchFailure createShardFailureTestItem() {
+        String randomMessage = randomAlphaOfLengthBetween(3, 20);
+        Exception ex = new ParsingException(0, 0, randomMessage, new IllegalArgumentException("some bad argument"));
+        SearchShardTarget searchShardTarget = null;
+        if (randomBoolean()) {
+            String nodeId = randomAlphaOfLengthBetween(5, 10);
+            String indexName = randomAlphaOfLengthBetween(5, 10);
+            searchShardTarget = new SearchShardTarget(nodeId,
+                new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), randomInt()), null, null);
+        }
+        return new ShardSearchFailure(ex, searchShardTarget);
+    }
+
+    private void assertEqualInstances(CountResponse expectedInstance, CountResponse newInstance) {
+        assertEquals(expectedInstance.getCount(), newInstance.getCount());
+        assertEquals(expectedInstance.status(), newInstance.status());
+        assertEquals(expectedInstance.isTerminatedEarly(), newInstance.isTerminatedEarly());
+        assertEquals(expectedInstance.getTotalShards(), newInstance.getTotalShards());
+        assertEquals(expectedInstance.getFailedShards(), newInstance.getFailedShards());
+        assertEquals(expectedInstance.getSkippedShards(), newInstance.getSkippedShards());
+        assertEquals(expectedInstance.getSuccessfulShards(), newInstance.getSuccessfulShards());
+        assertEquals(expectedInstance.getShardFailures().length, newInstance.getShardFailures().length);
+
+        ShardSearchFailure[] expectedFailures = expectedInstance.getShardFailures();
+        ShardSearchFailure[] newFailures = newInstance.getShardFailures();
+
+        for (int i = 0; i < newFailures.length; i++) {
+            ShardSearchFailure parsedFailure = newFailures[i];
+            ShardSearchFailure originalFailure = expectedFailures[i];
+            assertEquals(originalFailure.index(), parsedFailure.index());
+            assertEquals(originalFailure.shard(), parsedFailure.shard());
+            assertEquals(originalFailure.shardId(), parsedFailure.shardId());
+            String originalMsg = originalFailure.getCause().getMessage();
+            assertEquals(parsedFailure.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" +
+                originalMsg + "]");
+            String nestedMsg = originalFailure.getCause().getCause().getMessage();
+            assertEquals(parsedFailure.getCause().getCause().getMessage(),
+                "Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]");
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
index cb1890c7789a1..5530c82272698 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -1545,23 +1545,23 @@ public void afterBulk(long executionId, BulkRequest request,
     // Not entirely sure if _termvectors belongs to CRUD, and in the absence of a better place, will have it here
     public void testTermVectors() throws Exception {
         RestHighLevelClient client = highLevelClient();
-        CreateIndexRequest authorsRequest = new CreateIndexRequest("authors").mapping("doc", "user", "type=keyword");
+        CreateIndexRequest authorsRequest = new CreateIndexRequest("authors").mapping("_doc", "user", "type=keyword");
         CreateIndexResponse authorsResponse = client.indices().create(authorsRequest, RequestOptions.DEFAULT);
         assertTrue(authorsResponse.isAcknowledged());
-        client.index(new IndexRequest("index", "doc", "1").source("user", "kimchy"), RequestOptions.DEFAULT);
+        client.index(new IndexRequest("index", "_doc", "1").source("user", "kimchy"), RequestOptions.DEFAULT);
         Response refreshResponse = client().performRequest(new Request("POST", "/authors/_refresh"));
         assertEquals(200, refreshResponse.getStatusLine().getStatusCode());
 
         {
             // tag::term-vectors-request
-            TermVectorsRequest request = new TermVectorsRequest("authors", "doc", "1");
+            TermVectorsRequest request = new TermVectorsRequest("authors", "_doc", "1");
             request.setFields("user");
             // end::term-vectors-request
         }
 
         {
             // tag::term-vectors-request-artificial
-            TermVectorsRequest request = new TermVectorsRequest("authors", "doc");
+            TermVectorsRequest request = new TermVectorsRequest("authors", "_doc");
             XContentBuilder docBuilder = XContentFactory.jsonBuilder();
             docBuilder.startObject().field("user", "guest-user").endObject();
             request.setDoc(docBuilder); // <1>
@@ -1594,7 +1594,7 @@ public void testTermVectors() throws Exception {
             // end::term-vectors-request-optional-arguments
         }
 
-        TermVectorsRequest request = new TermVectorsRequest("authors", "doc", "1");
+        TermVectorsRequest request = new TermVectorsRequest("authors", "_doc", "1");
         request.setFields("user");
 
         // tag::term-vectors-execute
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index cf6a9622c0c72..90337ebf6053e 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -74,6 +74,8 @@
 import org.elasticsearch.client.ml.PutCalendarResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
+import org.elasticsearch.client.ml.PutFilterRequest;
+import org.elasticsearch.client.ml.PutFilterResponse;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.PutJobResponse;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
@@ -94,6 +96,7 @@
 import org.elasticsearch.client.ml.job.config.Detector;
 import org.elasticsearch.client.ml.job.config.Job;
 import org.elasticsearch.client.ml.job.config.JobUpdate;
+import org.elasticsearch.client.ml.job.config.MlFilter;
 import org.elasticsearch.client.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.client.ml.job.config.Operator;
 import org.elasticsearch.client.ml.job.config.RuleCondition;
@@ -2007,4 +2010,58 @@ public void onFailure(Exception e) {
 
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
     }
+
+    public void testCreateFilter() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::put-filter-config
+            MlFilter.Builder filterBuilder = MlFilter.builder("my_safe_domains") // <1>
+                .setDescription("A list of safe domains")   // <2>
+                .setItems("*.google.com", "wikipedia.org"); // <3>
+            // end::put-filter-config
+
+            // tag::put-filter-request
+            PutFilterRequest request = new PutFilterRequest(filterBuilder.build()); // <1>
+            // end::put-filter-request
+
+            // tag::put-filter-execute
+            PutFilterResponse response = client.machineLearning().putFilter(request, RequestOptions.DEFAULT);
+            // end::put-filter-execute
+
+            // tag::put-filter-response
+            MlFilter createdFilter = response.getResponse(); // <1>
+            // end::put-filter-response
+            assertThat(createdFilter.getId(), equalTo("my_safe_domains"));
+        }
+        {
+            MlFilter.Builder filterBuilder = MlFilter.builder("safe_domains_async")
+                .setDescription("A list of safe domains")
+                .setItems("*.google.com", "wikipedia.org");
+
+            PutFilterRequest request = new PutFilterRequest(filterBuilder.build());
+            // tag::put-filter-execute-listener
+            ActionListener<PutFilterResponse> listener = new ActionListener<PutFilterResponse>() {
+                @Override
+                public void onResponse(PutFilterResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::put-filter-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::put-filter-execute-async
+            client.machineLearning().putFilterAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::put-filter-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java
index e196dbc0229ef..e790ddc405eef 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java
@@ -41,6 +41,8 @@
 import org.elasticsearch.client.rollup.GetRollupJobResponse.JobWrapper;
 import org.elasticsearch.client.rollup.GetRollupJobResponse.RollupIndexerJobStats;
 import org.elasticsearch.client.rollup.GetRollupJobResponse.RollupJobStatus;
+import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
+import org.elasticsearch.client.rollup.DeleteRollupJobResponse;
 import org.elasticsearch.client.rollup.PutRollupJobRequest;
 import org.elasticsearch.client.rollup.PutRollupJobResponse;
 import org.elasticsearch.client.rollup.RollableIndexCaps;
@@ -451,4 +453,53 @@ private void waitForPendingRollupTasks() throws Exception {
             }
         });
     }
+
+    public void testDeleteRollupJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        String id = "job_2";
+
+        // tag::rollup-delete-job-request
+        DeleteRollupJobRequest request = new DeleteRollupJobRequest(id); // <1>
+        // end::rollup-delete-job-request
+        try {
+            // tag::rollup-delete-job-execute
+            DeleteRollupJobResponse response = client.rollup().deleteRollupJob(request, RequestOptions.DEFAULT);
+            // end::rollup-delete-job-execute
+
+            // tag::rollup-delete-job-response
+            response.isAcknowledged(); // <1>
+            // end::rollup-delete-job-response
+        } catch (Exception e) {
+            // Swallow any exception, this test does not test actually cancelling.
+        }
+
+
+
+        // tag::rollup-delete-job-execute-listener
+        ActionListener<DeleteRollupJobResponse> listener = new ActionListener<DeleteRollupJobResponse>() {
+            @Override
+            public void onResponse(DeleteRollupJobResponse response) {
+                boolean acknowledged = response.isAcknowledged(); // <1>
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // <2>
+            }
+        };
+        // end::rollup-delete-job-execute-listener
+
+        // Replace the empty listener by a blocking listener in test
+        final CountDownLatch latch = new CountDownLatch(1);
+        listener = new LatchedActionListener<>(listener, latch);
+
+        // tag::rollup-delete-job-execute-async
+        client.rollup().deleteRollupJobAsync(request, RequestOptions.DEFAULT, listener); // <1>
+        // end::rollup-delete-job-execute-async
+
+        assertTrue(latch.await(30L, TimeUnit.SECONDS));
+
+    }
+
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
index 8df7fd20a180b..e97620b50d4e6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java
@@ -49,6 +49,8 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.text.Text;
@@ -1284,4 +1286,124 @@ private void indexSearchTestData() throws IOException {
         assertSame(RestStatus.OK, bulkResponse.status());
         assertFalse(bulkResponse.hasFailures());
     }
+
+
+    @SuppressWarnings({"unused", "unchecked"})
+    public void testCount() throws Exception {
+        indexCountTestData();
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::count-request-basic
+            CountRequest countRequest = new CountRequest(); // <1>
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // <2>
+            searchSourceBuilder.query(QueryBuilders.matchAllQuery()); // <3>
+            countRequest.source(searchSourceBuilder); // <4>
+            // end::count-request-basic
+        }
+        {
+            // tag::count-request-indices-types
+            CountRequest countRequest = new CountRequest("blog"); // <1>
+            countRequest.types("doc"); // <2>
+            // end::count-request-indices-types
+            // tag::count-request-routing
+            countRequest.routing("routing"); // <1>
+            // end::count-request-routing
+            // tag::count-request-indicesOptions
+            countRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1>
+            // end::count-request-indicesOptions
+            // tag::count-request-preference
+            countRequest.preference("_local"); // <1>
+            // end::count-request-preference
+            assertNotNull(client.count(countRequest, RequestOptions.DEFAULT));
+        }
+        {
+            // tag::count-source-basics
+            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // <1>
+            sourceBuilder.query(QueryBuilders.termQuery("user", "kimchy")); // <2>
+            // end::count-source-basics
+
+            // tag::count-source-setter
+            CountRequest countRequest = new CountRequest();
+            countRequest.indices("blog", "author");
+            countRequest.source(sourceBuilder);
+            // end::count-source-setter
+
+            // tag::count-execute
+            CountResponse countResponse = client
+                .count(countRequest, RequestOptions.DEFAULT);
+            // end::count-execute
+
+            // tag::count-execute-listener
+            ActionListener<CountResponse> listener =
+                new ActionListener<CountResponse>() {
+
+                    @Override
+                    public void onResponse(CountResponse countResponse) {
+                        // <1>
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // <2>
+                    }
+                };
+            // end::count-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::count-execute-async
+            client.countAsync(countRequest, RequestOptions.DEFAULT, listener); // <1>
+            // end::count-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+
+            // tag::count-response-1
+            long count = countResponse.getCount();
+            RestStatus status = countResponse.status();
+            Boolean terminatedEarly = countResponse.isTerminatedEarly();
+            // end::count-response-1
+
+            // tag::count-response-2
+            int totalShards = countResponse.getTotalShards();
+            int skippedShards = countResponse.getSkippedShards();
+            int successfulShards = countResponse.getSuccessfulShards();
+            int failedShards = countResponse.getFailedShards();
+            for (ShardSearchFailure failure : countResponse.getShardFailures()) {
+                // failures should be handled here
+            }
+            // end::count-response-2
+            assertNotNull(countResponse);
+            assertEquals(4, countResponse.getCount());
+        }
+    }
+
+    private static void indexCountTestData() throws IOException {
+        CreateIndexRequest authorsRequest = new CreateIndexRequest("author")
+            .mapping("doc", "user", "type=keyword,doc_values=false");
+        CreateIndexResponse authorsResponse = highLevelClient().indices().create(authorsRequest, RequestOptions.DEFAULT);
+        assertTrue(authorsResponse.isAcknowledged());
+
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(new IndexRequest("blog", "doc", "1")
+            .source(XContentType.JSON, "title", "Doubling Down on Open?", "user",
+                Collections.singletonList("kimchy"), "innerObject", Collections.singletonMap("key", "value")));
+        bulkRequest.add(new IndexRequest("blog", "doc", "2")
+            .source(XContentType.JSON, "title", "Swiftype Joins Forces with Elastic", "user",
+                Arrays.asList("kimchy", "matt"), "innerObject", Collections.singletonMap("key", "value")));
+        bulkRequest.add(new IndexRequest("blog", "doc", "3")
+            .source(XContentType.JSON, "title", "On Net Neutrality", "user",
+                Arrays.asList("tyler", "kimchy"), "innerObject", Collections.singletonMap("key", "value")));
+
+        bulkRequest.add(new IndexRequest("author", "doc", "1")
+            .source(XContentType.JSON, "user", "kimchy"));
+
+
+        bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT);
+        assertSame(RestStatus.OK, bulkResponse.status());
+        assertFalse(bulkResponse.hasFailures());
+    }
+
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
index 7c37c7ef50a7a..fdcaa56fa004d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
@@ -29,7 +29,10 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.AuthenticateResponse;
 import org.elasticsearch.client.security.ChangePasswordRequest;
+import org.elasticsearch.client.security.ClearRealmCacheRequest;
+import org.elasticsearch.client.security.ClearRealmCacheResponse;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
@@ -50,10 +53,11 @@
 import org.elasticsearch.client.security.PutUserRequest;
 import org.elasticsearch.client.security.PutUserResponse;
 import org.elasticsearch.client.security.RefreshPolicy;
-import org.elasticsearch.client.security.support.CertificateInfo;
 import org.elasticsearch.client.security.support.expressiondsl.RoleMapperExpression;
-import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression;
 import org.elasticsearch.client.security.support.expressiondsl.fields.FieldRoleMapperExpression;
+import org.elasticsearch.client.security.user.User;
+import org.elasticsearch.client.security.support.CertificateInfo;
+import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
@@ -67,13 +71,14 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.not;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isIn;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
 
@@ -379,6 +384,97 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testAuthenticate() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::authenticate-execute
+            AuthenticateResponse response = client.security().authenticate(RequestOptions.DEFAULT);
+            //end::authenticate-execute
+
+            //tag::authenticate-response
+            User user = response.getUser(); // <1>
+            boolean enabled = response.enabled(); // <2>
+            //end::authenticate-response
+
+            assertThat(user.username(), is("test_user"));
+            assertThat(user.roles(), contains(new String[]{"superuser"}));
+            assertThat(user.fullName(), nullValue());
+            assertThat(user.email(), nullValue());
+            assertThat(user.metadata().isEmpty(), is(true));
+            assertThat(enabled, is(true));
+        }
+
+        {
+            // tag::authenticate-execute-listener
+            ActionListener<AuthenticateResponse> listener = new ActionListener<AuthenticateResponse>() {
+                @Override
+                public void onResponse(AuthenticateResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::authenticate-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::authenticate-execute-async
+            client.security().authenticateAsync(RequestOptions.DEFAULT, listener); // <1>
+            // end::authenticate-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testClearRealmCache() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            //tag::clear-realm-cache-request
+            ClearRealmCacheRequest request = new ClearRealmCacheRequest(Collections.emptyList(), Collections.emptyList());
+            //end::clear-realm-cache-request
+            //tag::clear-realm-cache-execute
+            ClearRealmCacheResponse response = client.security().clearRealmCache(request, RequestOptions.DEFAULT);
+            //end::clear-realm-cache-execute
+
+            assertNotNull(response);
+            assertThat(response.getNodes(), not(empty()));
+
+            //tag::clear-realm-cache-response
+            List<ClearRealmCacheResponse.Node> nodes = response.getNodes(); // <1>
+            //end::clear-realm-cache-response
+        }
+        {
+            //tag::clear-realm-cache-execute-listener
+            ClearRealmCacheRequest request = new ClearRealmCacheRequest(Collections.emptyList(), Collections.emptyList());
+            ActionListener<ClearRealmCacheResponse> listener = new ActionListener<ClearRealmCacheResponse>() {
+                @Override
+                public void onResponse(ClearRealmCacheResponse clearRealmCacheResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            //end::clear-realm-cache-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::clear-realm-cache-execute-async
+            client.security().clearRealmCacheAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::clear-realm-cache-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
 
     public void testClearRolesCache() throws Exception {
         RestHighLevelClient client = highLevelClient();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/graph/GraphExploreResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/graph/GraphExploreResponseTests.java
index b945bf4b7f5fd..5348337f87104 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/graph/GraphExploreResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/graph/GraphExploreResponseTests.java
@@ -86,7 +86,7 @@ private static   GraphExploreResponse createTestInstanceWithFailures() {
 
     @Override
     protected  GraphExploreResponse doParseInstance(XContentParser parser) throws IOException {
-        return GraphExploreResponse.fromXContext(parser);
+        return GraphExploreResponse.fromXContent(parser);
     }
 
     @Override
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java
new file mode 100644
index 0000000000000..e44eb0da0e188
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/AllocateActionTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class AllocateActionTests extends AbstractXContentTestCase<AllocateAction> {
+
+    @Override
+    protected AllocateAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static AllocateAction randomInstance() {
+        boolean hasAtLeastOneMap = false;
+        Map<String, String> includes;
+        if (randomBoolean()) {
+            includes = randomMap(1, 100);
+            hasAtLeastOneMap = true;
+        } else {
+            includes = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Map<String, String> excludes;
+        if (randomBoolean()) {
+            hasAtLeastOneMap = true;
+            excludes = randomMap(1, 100);
+        } else {
+            excludes = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Map<String, String> requires;
+        if (hasAtLeastOneMap == false || randomBoolean()) {
+            requires = randomMap(1, 100);
+        } else {
+            requires = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Integer numberOfReplicas = randomBoolean() ? null : randomIntBetween(0, 10);
+        return new AllocateAction(numberOfReplicas, includes, excludes, requires);
+    }
+
+    @Override
+    protected AllocateAction doParseInstance(XContentParser parser) {
+        return AllocateAction.parse(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testAllMapsNullOrEmpty() {
+        Map<String, String> include = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> exclude = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> require = randomBoolean() ? null : Collections.emptyMap();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> new AllocateAction(null, include, exclude, require));
+        assertEquals("At least one of " + AllocateAction.INCLUDE_FIELD.getPreferredName() + ", "
+            + AllocateAction.EXCLUDE_FIELD.getPreferredName() + " or " + AllocateAction.REQUIRE_FIELD.getPreferredName()
+            + "must contain attributes for action " + AllocateAction.NAME, exception.getMessage());
+    }
+
+    public void testInvalidNumberOfReplicas() {
+        Map<String, String> include = randomMap(1, 5);
+        Map<String, String> exclude = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> require = randomBoolean() ? null : Collections.emptyMap();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> new AllocateAction(randomIntBetween(-1000, -1), include, exclude, require));
+        assertEquals("[" + AllocateAction.NUMBER_OF_REPLICAS_FIELD.getPreferredName() + "] must be >= 0", exception.getMessage());
+    }
+
+    public static Map<String, String> randomMap(int minEntries, int maxEntries) {
+        Map<String, String> map = new HashMap<>();
+        int numIncludes = randomIntBetween(minEntries, maxEntries);
+        for (int i = 0; i < numIncludes; i++) {
+            map.put(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        }
+        return map;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java
new file mode 100644
index 0000000000000..fb7deb97a2787
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteActionTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+public class DeleteActionTests extends AbstractXContentTestCase<DeleteAction> {
+
+    @Override
+    protected DeleteAction createTestInstance() {
+        return new DeleteAction();
+    }
+
+    @Override
+    protected DeleteAction doParseInstance(XContentParser parser) {
+        return DeleteAction.parse(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequestTests.java
new file mode 100644
index 0000000000000..01f6288d81d4b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/DeleteLifecyclePolicyRequestTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class DeleteLifecyclePolicyRequestTests extends ESTestCase {
+
+    private DeleteLifecyclePolicyRequest createTestInstance() {
+        return new DeleteLifecyclePolicyRequest(randomAlphaOfLengthBetween(2, 20));
+    }
+
+    public void testValidate() {
+        DeleteLifecyclePolicyRequest req = createTestInstance();
+        assertFalse(req.validate().isPresent());
+
+    }
+
+    public void testValidationFailure() {
+        expectThrows(IllegalArgumentException.class, () -> new DeleteLifecyclePolicyRequest(randomFrom("", null)));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java
new file mode 100644
index 0000000000000..933503e629b06
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.Arrays;
+
+public class ExplainLifecycleRequestTests extends ESTestCase {
+
+    public void testEqualsAndHashcode() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copy, this::mutateInstance);
+    }
+
+    private ExplainLifecycleRequest createTestInstance() {
+        ExplainLifecycleRequest request = new ExplainLifecycleRequest();
+        if (randomBoolean()) {
+            request.indices(generateRandomStringArray(20, 20, false, true));
+        }
+        if (randomBoolean()) {
+            IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            request.indicesOptions(indicesOptions);
+        }
+        return request;
+    }
+
+    private ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance) {
+        String[] indices = instance.indices();
+        IndicesOptions indicesOptions = instance.indicesOptions();
+        switch (between(0, 1)) {
+        case 0:
+            indices = randomValueOtherThanMany(i -> Arrays.equals(i, instance.indices()),
+                    () -> generateRandomStringArray(20, 10, false, true));
+            break;
+        case 1:
+            indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest();
+        newRequest.indices(indices);
+        newRequest.indicesOptions(indicesOptions);
+        return newRequest;
+    }
+
+    private ExplainLifecycleRequest copy(ExplainLifecycleRequest original) {
+        ExplainLifecycleRequest copy = new ExplainLifecycleRequest();
+        copy.indices(original.indices());
+        copy.indicesOptions(original.indicesOptions());
+        return copy;
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponseTests.java
new file mode 100644
index 0000000000000..26eacb04b024f
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleResponseTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ExplainLifecycleResponseTests extends AbstractXContentTestCase<ExplainLifecycleResponse> {
+
+    @Override
+    protected ExplainLifecycleResponse createTestInstance() {
+        Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(0, 2); i++) {
+            IndexLifecycleExplainResponse indexResponse = IndexExplainResponseTests.randomIndexExplainResponse();
+            indexResponses.put(indexResponse.getIndex(), indexResponse);
+        }
+        return new ExplainLifecycleResponse(indexResponses);
+    }
+
+    @Override
+    protected ExplainLifecycleResponse doParseInstance(XContentParser parser) throws IOException {
+        return ExplainLifecycleResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java
new file mode 100644
index 0000000000000..16fafcfa24015
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ForceMergeActionTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ForceMergeActionTests extends AbstractXContentTestCase<ForceMergeAction> {
+
+    @Override
+    protected ForceMergeAction doParseInstance(XContentParser parser) {
+        return ForceMergeAction.parse(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected ForceMergeAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static ForceMergeAction randomInstance() {
+        return new ForceMergeAction(randomIntBetween(1, 100));
+    }
+
+    public void testMissingMaxNumSegments() throws IOException {
+        BytesReference emptyObject = BytesReference.bytes(JsonXContent.contentBuilder().startObject().endObject());
+        XContentParser parser = XContentHelper.createParser(null, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            emptyObject, XContentType.JSON);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ForceMergeAction.parse(parser));
+        assertThat(e.getMessage(), equalTo("Required [max_num_segments]"));
+    }
+
+    public void testInvalidNegativeSegmentNumber() {
+        Exception r = expectThrows(IllegalArgumentException.class, () -> new ForceMergeAction(randomIntBetween(-10, 0)));
+        assertThat(r.getMessage(), equalTo("[max_num_segments] must be a positive integer"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequestTests.java
new file mode 100644
index 0000000000000..06d28207ce93a
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyRequestTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class GetLifecyclePolicyRequestTests extends ESTestCase {
+
+    private GetLifecyclePolicyRequest createTestInstance() {
+        int numPolicies = randomIntBetween(0, 10);
+        String[] policyNames = new String[numPolicies];
+        for (int i = 0; i < numPolicies; i++) {
+            policyNames[i] = "policy-" + randomAlphaOfLengthBetween(2, 5);
+        }
+        return new GetLifecyclePolicyRequest(policyNames);
+    }
+
+    public void testValidation() {
+        GetLifecyclePolicyRequest request = createTestInstance();
+        assertFalse(request.validate().isPresent());
+    }
+
+    public void testNullPolicyNameShouldFail() {
+        expectThrows(IllegalArgumentException.class,
+            () -> new GetLifecyclePolicyRequest(randomAlphaOfLengthBetween(2,20), null, randomAlphaOfLengthBetween(2,20)));
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java
new file mode 100644
index 0000000000000..89dfbb8635332
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/GetLifecyclePolicyResponseTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
+
+public class GetLifecyclePolicyResponseTests extends AbstractXContentTestCase<GetLifecyclePolicyResponse> {
+
+    @Override
+    protected GetLifecyclePolicyResponse createTestInstance() {
+        int numPolicies = randomIntBetween(1, 10);
+        ImmutableOpenMap.Builder<String, LifecyclePolicyMetadata> policies = ImmutableOpenMap.builder();
+        for (int i = 0; i < numPolicies; i++) {
+            String policyName = "policy-" + randomAlphaOfLengthBetween(2, 5);
+            LifecyclePolicy policy = createRandomPolicy(policyName);
+            policies.put(policyName, new LifecyclePolicyMetadata(policy, randomLong(), randomLong()));
+        }
+        return new GetLifecyclePolicyResponse(policies.build());
+    }
+
+    @Override
+    protected GetLifecyclePolicyResponse doParseInstance(XContentParser parser) throws IOException {
+        return GetLifecyclePolicyResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexExplainResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexExplainResponseTests.java
new file mode 100644
index 0000000000000..fb7e73ee62191
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexExplainResponseTests.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class IndexExplainResponseTests extends AbstractXContentTestCase<IndexLifecycleExplainResponse> {
+
+    static IndexLifecycleExplainResponse randomIndexExplainResponse() {
+        if (frequently()) {
+            return randomManagedIndexExplainResponse();
+        } else {
+            return randomUnmanagedIndexExplainResponse();
+        }
+    }
+
+    private static IndexLifecycleExplainResponse randomUnmanagedIndexExplainResponse() {
+        return IndexLifecycleExplainResponse.newUnmanagedIndexResponse(randomAlphaOfLength(10));
+    }
+
+    private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse() {
+        return IndexLifecycleExplainResponse.newManagedIndexResponse(randomAlphaOfLength(10), randomAlphaOfLength(10),
+                randomNonNegativeLong(), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10),
+                randomBoolean() ? null : randomAlphaOfLength(10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
+                randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""));
+    }
+
+    @Override
+    protected IndexLifecycleExplainResponse createTestInstance() {
+        return randomIndexExplainResponse();
+    }
+
+    @Override
+    protected IndexLifecycleExplainResponse doParseInstance(XContentParser parser) throws IOException {
+        return IndexLifecycleExplainResponse.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    private static class RandomStepInfo implements ToXContentObject {
+
+        private final String key;
+        private final String value;
+
+        RandomStepInfo(Supplier<String> randomStringSupplier) {
+            this.key = randomStringSupplier.get();
+            this.value = randomStringSupplier.get();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(key, value);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(key, value);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            RandomStepInfo other = (RandomStepInfo) obj;
+            return Objects.equals(key, other.key) && Objects.equals(value, other.value);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java
new file mode 100644
index 0000000000000..144039b8995c6
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponseTests.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.CoreMatchers;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.stream.Collectors;
+
+public class LifecycleManagementStatusResponseTests extends ESTestCase {
+
+    public void testAllValidStatuses() {
+        EnumSet.allOf(OperationMode.class)
+            .forEach(e -> assertEquals(new LifecycleManagementStatusResponse(e.name()).getOperationMode(), e));
+    }
+
+    public void testXContent() throws IOException {
+        XContentType xContentType = XContentType.JSON;
+        String mode = randomFrom(EnumSet.allOf(OperationMode.class)
+            .stream().map(Enum::name).collect(Collectors.toList()));
+        XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{\"operation_mode\" : \"" + mode + "\"}");
+        assertEquals(LifecycleManagementStatusResponse.fromXContent(parser).getOperationMode(), OperationMode.fromString(mode));
+    }
+
+    public void testXContentInvalid() throws IOException {
+        XContentType xContentType = XContentType.JSON;
+        String mode = randomAlphaOfLength(10);
+        XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{\"operation_mode\" : \"" + mode + "\"}");
+        Exception e = expectThrows(IllegalArgumentException.class, () -> LifecycleManagementStatusResponse.fromXContent(parser));
+        assertThat(e.getMessage(), CoreMatchers.containsString("failed to parse field [operation_mode]"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java
new file mode 100644
index 0000000000000..548ba366b640e
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadataTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
+
+public class LifecyclePolicyMetadataTests extends AbstractXContentTestCase<LifecyclePolicyMetadata> {
+
+    private String policyName;
+
+    @Override
+    protected LifecyclePolicyMetadata createTestInstance() {
+        policyName = randomAlphaOfLengthBetween(5,20);
+        LifecyclePolicy policy = createRandomPolicy(policyName);
+        return new LifecyclePolicyMetadata(policy, randomLong(), randomLong());
+    }
+
+    @Override
+    protected LifecyclePolicyMetadata doParseInstance(XContentParser parser) throws IOException {
+        return LifecyclePolicyMetadata.parse(parser, policyName);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java
new file mode 100644
index 0000000000000..024cb13d8df37
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyTests.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LifecyclePolicyTests extends AbstractXContentTestCase<LifecyclePolicy> {
+    private static final Set<String> VALID_HOT_ACTIONS = Sets.newHashSet(RolloverAction.NAME);
+    private static final Set<String> VALID_WARM_ACTIONS = Sets.newHashSet(AllocateAction.NAME, ForceMergeAction.NAME,
+        ReadOnlyAction.NAME, ShrinkAction.NAME);
+    private static final Set<String> VALID_COLD_ACTIONS = Sets.newHashSet(AllocateAction.NAME);
+    private static final Set<String> VALID_DELETE_ACTIONS = Sets.newHashSet(DeleteAction.NAME);
+
+    private String lifecycleName;
+
+    @Override
+    protected LifecyclePolicy doParseInstance(XContentParser parser) {
+        return LifecyclePolicy.parse(parser, lifecycleName);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected LifecyclePolicy createTestInstance() {
+        lifecycleName = randomAlphaOfLength(5);
+        return createRandomPolicy(lifecycleName);
+    }
+
+    public void testValidatePhases() {
+        boolean invalid = randomBoolean();
+        String phaseName = randomFrom("hot", "warm", "cold", "delete");
+        if (invalid) {
+            phaseName += randomAlphaOfLength(5);
+        }
+        Map<String, Phase> phases = Collections.singletonMap(phaseName,
+            new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap()));
+        if (invalid) {
+            Exception e = expectThrows(IllegalArgumentException.class, () -> new LifecyclePolicy(lifecycleName, phases));
+            assertThat(e.getMessage(), equalTo("Lifecycle does not support phase [" + phaseName + "]"));
+        } else {
+            new LifecyclePolicy(lifecycleName, phases);
+        }
+    }
+
+    public void testValidateHotPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = randomSubsetOf(VALID_HOT_ACTIONS)
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("allocate", "forcemerge", "delete", "shrink"));
+            actions.put(invalidAction.getName(), invalidAction);
+        }
+        Map<String, Phase> hotPhase = Collections.singletonMap("hot",
+            new Phase("hot", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                () -> new LifecyclePolicy(lifecycleName, hotPhase));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getName() + "] defined in phase [hot]"));
+        } else {
+            new LifecyclePolicy(lifecycleName, hotPhase);
+        }
+    }
+
+    public void testValidateWarmPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = randomSubsetOf(VALID_WARM_ACTIONS)
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("rollover", "delete"));
+            actions.put(invalidAction.getName(), invalidAction);
+        }
+        Map<String, Phase> warmPhase = Collections.singletonMap("warm",
+            new Phase("warm", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                () -> new LifecyclePolicy(lifecycleName, warmPhase));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getName() + "] defined in phase [warm]"));
+        } else {
+            new LifecyclePolicy(lifecycleName, warmPhase);
+        }
+    }
+
+    public void testValidateColdPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = randomSubsetOf(VALID_COLD_ACTIONS)
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("rollover", "delete", "forcemerge", "shrink"));
+            actions.put(invalidAction.getName(), invalidAction);
+        }
+        Map<String, Phase> coldPhase = Collections.singletonMap("cold",
+            new Phase("cold", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                () -> new LifecyclePolicy(lifecycleName, coldPhase));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getName() + "] defined in phase [cold]"));
+        } else {
+            new LifecyclePolicy(lifecycleName, coldPhase);
+        }
+    }
+
+    public void testValidateDeletePhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = VALID_DELETE_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("allocate", "rollover", "forcemerge", "shrink"));
+            actions.put(invalidAction.getName(), invalidAction);
+        }
+        Map<String, Phase> deletePhase = Collections.singletonMap("delete",
+            new Phase("delete", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                () -> new LifecyclePolicy(lifecycleName, deletePhase));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getName() + "] defined in phase [delete]"));
+        } else {
+            new LifecyclePolicy(lifecycleName, deletePhase);
+        }
+    }
+
+    public static LifecyclePolicy createRandomPolicy(String lifecycleName) {
+        List<String> phaseNames = randomSubsetOf(Arrays.asList("hot", "warm", "cold", "delete"));
+        Map<String, Phase> phases = new HashMap<>(phaseNames.size());
+        Function<String, Set<String>> validActions = (phase) ->  {
+            switch (phase) {
+                case "hot":
+                    return VALID_HOT_ACTIONS;
+                case "warm":
+                    return VALID_WARM_ACTIONS;
+                case "cold":
+                    return VALID_COLD_ACTIONS;
+                case "delete":
+                    return VALID_DELETE_ACTIONS;
+                default:
+                    throw new IllegalArgumentException("invalid phase [" + phase + "]");
+            }};
+        Function<String, LifecycleAction> randomAction = (action) ->  {
+            switch (action) {
+                case AllocateAction.NAME:
+                    return AllocateActionTests.randomInstance();
+                case DeleteAction.NAME:
+                    return new DeleteAction();
+                case ForceMergeAction.NAME:
+                    return ForceMergeActionTests.randomInstance();
+                case ReadOnlyAction.NAME:
+                    return new ReadOnlyAction();
+                case RolloverAction.NAME:
+                    return RolloverActionTests.randomInstance();
+                case ShrinkAction.NAME:
+                    return ShrinkActionTests.randomInstance();
+                default:
+                    throw new IllegalArgumentException("invalid action [" + action + "]");
+            }};
+        for (String phase : phaseNames) {
+            TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            List<String> actionNames = randomSubsetOf(validActions.apply(phase));
+            for (String action : actionNames) {
+                actions.put(action, randomAction.apply(action));
+            }
+            phases.put(phase, new Phase(phase, after, actions));
+        }
+        return new LifecyclePolicy(lifecycleName, phases);
+    }
+
+    private LifecycleAction getTestAction(String actionName) {
+        switch (actionName) {
+            case AllocateAction.NAME:
+                return AllocateActionTests.randomInstance();
+            case DeleteAction.NAME:
+                return new DeleteAction();
+            case ForceMergeAction.NAME:
+                return ForceMergeActionTests.randomInstance();
+            case ReadOnlyAction.NAME:
+                return new ReadOnlyAction();
+            case RolloverAction.NAME:
+                return RolloverActionTests.randomInstance();
+            case ShrinkAction.NAME:
+                return ShrinkActionTests.randomInstance();
+            default:
+                throw new IllegalArgumentException("unsupported phase action [" + actionName + "]");
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/OperationModeTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/OperationModeTests.java
new file mode 100644
index 0000000000000..27651ba4a8c41
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/OperationModeTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.CoreMatchers;
+
+import java.util.EnumSet;
+
+public class OperationModeTests extends ESTestCase {
+
+    public void testIsValidChange() {
+        assertFalse(OperationMode.RUNNING.isValidChange(OperationMode.RUNNING));
+        assertTrue(OperationMode.RUNNING.isValidChange(OperationMode.STOPPING));
+        assertFalse(OperationMode.RUNNING.isValidChange(OperationMode.STOPPED));
+
+        assertTrue(OperationMode.STOPPING.isValidChange(OperationMode.RUNNING));
+        assertFalse(OperationMode.STOPPING.isValidChange(OperationMode.STOPPING));
+        assertTrue(OperationMode.STOPPING.isValidChange(OperationMode.STOPPED));
+
+        assertTrue(OperationMode.STOPPED.isValidChange(OperationMode.RUNNING));
+        assertFalse(OperationMode.STOPPED.isValidChange(OperationMode.STOPPING));
+        assertFalse(OperationMode.STOPPED.isValidChange(OperationMode.STOPPED));
+    }
+
+    public void testFromName() {
+        EnumSet.allOf(OperationMode.class).forEach(e -> assertEquals(OperationMode.fromString(e.name()), e));
+    }
+
+    public void testFromNameInvalid() {
+        String invalidName = randomAlphaOfLength(10);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> OperationMode.fromString(invalidName));
+        assertThat(e.getMessage(), CoreMatchers.containsString(invalidName + " is not a valid operation_mode"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java
new file mode 100644
index 0000000000000..0db9b56aea93c
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfoTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PhaseExecutionInfoTests extends AbstractXContentTestCase<PhaseExecutionInfo> {
+
+    static PhaseExecutionInfo randomPhaseExecutionInfo(String phaseName) {
+        return new PhaseExecutionInfo(randomAlphaOfLength(5), PhaseTests.randomPhase(phaseName),
+            randomNonNegativeLong(), randomNonNegativeLong());
+    }
+
+    String phaseName;
+
+    @Before
+    public void setupPhaseName() {
+        phaseName = randomAlphaOfLength(7);
+    }
+
+    @Override
+    protected PhaseExecutionInfo createTestInstance() {
+        return randomPhaseExecutionInfo(phaseName);
+    }
+
+    @Override
+    protected PhaseExecutionInfo doParseInstance(XContentParser parser) throws IOException {
+        return PhaseExecutionInfo.parse(parser, phaseName);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java
new file mode 100644
index 0000000000000..3b4fc2fec6059
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PhaseTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class PhaseTests extends AbstractXContentTestCase<Phase> {
+    private String phaseName;
+
+    @Before
+    public void setup() {
+        phaseName = randomAlphaOfLength(20);
+    }
+
+    @Override
+    protected Phase createTestInstance() {
+        return randomPhase(phaseName);
+    }
+
+    static Phase randomPhase(String phaseName) {
+        TimeValue after = null;
+        if (randomBoolean()) {
+            after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+        }
+        Map<String, LifecycleAction> actions = Collections.emptyMap();
+        if (randomBoolean()) {
+            actions = Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
+        }
+        return new Phase(phaseName, after, actions);
+    }
+
+    @Override
+    protected Phase doParseInstance(XContentParser parser) {
+        return Phase.parse(parser, phaseName);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testDefaultAfter() {
+        Phase phase = new Phase(randomAlphaOfLength(20), null, Collections.emptyMap());
+        assertEquals(TimeValue.ZERO, phase.getMinimumAge());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequestTests.java
new file mode 100644
index 0000000000000..26cfe1946ac4d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/PutLifecyclePolicyRequestTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.client.indexlifecycle.LifecyclePolicyTests.createRandomPolicy;
+
+public class PutLifecyclePolicyRequestTests extends ESTestCase {
+
+    private PutLifecyclePolicyRequest createTestInstance() {
+        return new PutLifecyclePolicyRequest(createRandomPolicy(randomAlphaOfLengthBetween(5, 20)));
+    }
+
+    public void testValidation() {
+        PutLifecyclePolicyRequest req = createTestInstance();
+        assertFalse(req.validate().isPresent());
+    }
+
+    public void testNullPolicy() {
+        expectThrows(IllegalArgumentException.class, () -> new PutLifecyclePolicyRequest(null));
+    }
+
+    public void testNullPolicyName() {
+        expectThrows(IllegalArgumentException.class, () -> new PutLifecyclePolicyRequest(createRandomPolicy(randomFrom("", null))));
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java
new file mode 100644
index 0000000000000..bf57478425cc9
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ReadOnlyActionTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+public class ReadOnlyActionTests extends AbstractXContentTestCase<ReadOnlyAction> {
+
+    @Override
+    protected ReadOnlyAction doParseInstance(XContentParser parser) {
+        return ReadOnlyAction.parse(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected ReadOnlyAction createTestInstance() {
+        return new ReadOnlyAction();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java
new file mode 100644
index 0000000000000..d5ccabc748df5
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+
+public class RemoveIndexLifecyclePolicyRequestTests extends ESTestCase {
+
+    public void testNullIndices() {
+        expectThrows(NullPointerException.class, () -> new RemoveIndexLifecyclePolicyRequest(null));
+    }
+
+    public void testNullIndicesOptions() {
+        expectThrows(NullPointerException.class, () -> new RemoveIndexLifecyclePolicyRequest(Collections.emptyList(), null));
+    }
+
+    public void testValidate() {
+        RemoveIndexLifecyclePolicyRequest request = new RemoveIndexLifecyclePolicyRequest(Collections.emptyList());
+        assertFalse(request.validate().isPresent());
+    }
+
+    protected RemoveIndexLifecyclePolicyRequest createInstance() {
+        if (randomBoolean()) {
+            return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)),
+                    IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+        } else {
+            return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)));
+        }
+    }
+
+    private RemoveIndexLifecyclePolicyRequest copyInstance(RemoveIndexLifecyclePolicyRequest req) {
+        return new RemoveIndexLifecyclePolicyRequest(new ArrayList<>(req.indices()), IndicesOptions.fromOptions(
+                req.indicesOptions().ignoreUnavailable(), req.indicesOptions().allowNoIndices(),
+                req.indicesOptions().expandWildcardsOpen(), req.indicesOptions().expandWildcardsClosed(),
+                req.indicesOptions().allowAliasesToMultipleIndices(), req.indicesOptions().forbidClosedIndices(),
+                req.indicesOptions().ignoreAliases(), req.indicesOptions().ignoreThrottled()));
+    }
+
+    private RemoveIndexLifecyclePolicyRequest mutateInstance(RemoveIndexLifecyclePolicyRequest req) {
+        if (randomBoolean()) {
+            return new RemoveIndexLifecyclePolicyRequest(req.indices(),
+                    randomValueOtherThan(req.indicesOptions(), () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())));
+        } else {
+            return new RemoveIndexLifecyclePolicyRequest(
+                    randomValueOtherThan(req.indices(), () -> Arrays.asList(generateRandomStringArray(20, 20, false))),
+                    req.indicesOptions());
+        }
+    }
+
+    public void testEqualsAndHashCode() {
+        for (int count = 0; count < 100; ++count) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponseTests.java
new file mode 100644
index 0000000000000..1f99a2dfdfac4
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyResponseTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public class RemoveIndexLifecyclePolicyResponseTests extends ESTestCase {
+
+    private void toXContent(RemoveIndexLifecyclePolicyResponse response, XContentBuilder builder) throws IOException {
+        builder.startObject();
+        builder.field(RemoveIndexLifecyclePolicyResponse.HAS_FAILURES_FIELD.getPreferredName(), response.hasFailures());
+        builder.field(RemoveIndexLifecyclePolicyResponse.FAILED_INDEXES_FIELD.getPreferredName(), response.getFailedIndexes());
+        builder.endObject();
+    }
+
+    private RemoveIndexLifecyclePolicyResponse createInstance() {
+        List<String> failedIndexes = Arrays.asList(generateRandomStringArray(20, 20, false));
+        return new RemoveIndexLifecyclePolicyResponse(failedIndexes);
+    }
+
+    private RemoveIndexLifecyclePolicyResponse copyInstance(RemoveIndexLifecyclePolicyResponse req) {
+        return new RemoveIndexLifecyclePolicyResponse(new ArrayList<>(req.getFailedIndexes()));
+    }
+
+    private RemoveIndexLifecyclePolicyResponse mutateInstance(RemoveIndexLifecyclePolicyResponse req) {
+        return new RemoveIndexLifecyclePolicyResponse(randomValueOtherThan(req.getFailedIndexes(),
+                () -> Arrays.asList(generateRandomStringArray(20, 20, false))));
+    }
+
+    public void testFromXContent() throws IOException {
+        xContentTester(
+                this::createParser,
+                this::createInstance,
+                this::toXContent,
+                RemoveIndexLifecyclePolicyResponse::fromXContent)
+                .supportsUnknownFields(true)
+                .test();
+    }
+
+    public void testNullFailedIndices() {
+        IllegalArgumentException exception =
+                expectThrows(IllegalArgumentException.class, () -> new RemoveIndexLifecyclePolicyResponse(null));
+        assertEquals("failed_indexes cannot be null", exception.getMessage());
+    }
+
+    public void testHasFailures() {
+        RemoveIndexLifecyclePolicyResponse response = new RemoveIndexLifecyclePolicyResponse(new ArrayList<>());
+        assertFalse(response.hasFailures());
+        assertEquals(Collections.emptyList(), response.getFailedIndexes());
+
+        int size = randomIntBetween(1, 10);
+        List<String> failedIndexes = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            failedIndexes.add(randomAlphaOfLength(20));
+        }
+        response = new RemoveIndexLifecyclePolicyResponse(failedIndexes);
+        assertTrue(response.hasFailures());
+        assertEquals(failedIndexes, response.getFailedIndexes());
+    }
+
+    public void testEqualsAndHashCode() {
+        for (int count = 0; count < 100; ++count) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java
new file mode 100644
index 0000000000000..bbbdba37e5640
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+public class RolloverActionTests extends AbstractXContentTestCase<RolloverAction> {
+
+    @Override
+    protected RolloverAction doParseInstance(XContentParser parser) {
+        return RolloverAction.parse(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected RolloverAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static RolloverAction randomInstance() {
+        ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
+        ByteSizeValue maxSize = randomBoolean() ? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
+        Long maxDocs = randomBoolean() ? null : randomNonNegativeLong();
+        TimeValue maxAge = (maxDocs == null && maxSize == null || randomBoolean())
+            ? TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test")
+            : null;
+        return new RolloverAction(maxSize, maxAge, maxDocs);
+    }
+
+    public void testNoConditions() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> new RolloverAction(null, null, null));
+        assertEquals("At least one rollover condition must be set.", exception.getMessage());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java
new file mode 100644
index 0000000000000..adeec1ff825a9
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ShrinkActionTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShrinkActionTests extends AbstractXContentTestCase<ShrinkAction> {
+
+    @Override
+    protected ShrinkAction doParseInstance(XContentParser parser) throws IOException {
+        return ShrinkAction.parse(parser);
+    }
+
+    @Override
+    protected ShrinkAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static ShrinkAction randomInstance() {
+        return new ShrinkAction(randomIntBetween(1, 100));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testNonPositiveShardNumber() {
+        Exception e = expectThrows(Exception.class, () -> new ShrinkAction(randomIntBetween(-100, 0)));
+        assertThat(e.getMessage(), equalTo("[number_of_shards] must be greater than 0"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StartILMRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StartILMRequestTests.java
new file mode 100644
index 0000000000000..449ef7d1678eb
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StartILMRequestTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+public class StartILMRequestTests extends ESTestCase {
+
+    protected StartILMRequest createTestInstance() {
+        return new StartILMRequest();
+    }
+
+    public void testValidate() {
+        StartILMRequest request = createTestInstance();
+        assertFalse(request.validate().isPresent());
+    }
+
+    public void testEqualsAndHashcode() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), (original) -> createTestInstance());
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StopILMRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StopILMRequestTests.java
new file mode 100644
index 0000000000000..f1618f3f0f0e3
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/StopILMRequestTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+public class StopILMRequestTests extends ESTestCase {
+
+    protected StopILMRequest createTestInstance() {
+        return new StopILMRequest();
+    }
+
+    public void testValidate() {
+        StopILMRequest request = createTestInstance();
+        assertFalse(request.validate().isPresent());
+    }
+
+    public void testEqualsAndHashcode() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), (original) -> createTestInstance());
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java
deleted file mode 100644
index 8370a6ba9afed..0000000000000
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.client.license;
-
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.client.common.ProtocolUtils;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class StartBasicResponseTests extends ESTestCase {
-
-    public void testFromXContent() throws Exception {
-        StartBasicResponse.Status status = randomFrom(StartBasicResponse.Status.values());
-
-        boolean acknowledged = status != StartBasicResponse.Status.NEED_ACKNOWLEDGEMENT;
-        String acknowledgeMessage = null;
-        Map<String, String[]> ackMessages = Collections.emptyMap();
-        if (status != StartBasicResponse.Status.GENERATED_BASIC) {
-            acknowledgeMessage = randomAlphaOfLength(10);
-            ackMessages = randomAckMessages();
-        }
-
-        final StartBasicResponse startBasicResponse = new StartBasicResponse(status, ackMessages, acknowledgeMessage);
-
-        XContentType xContentType = randomFrom(XContentType.values());
-        XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
-
-        toXContent(startBasicResponse, builder);
-
-        final StartBasicResponse response = StartBasicResponse.fromXContent(createParser(builder));
-        assertThat(response.isAcknowledged(), equalTo(acknowledged));
-        assertThat(response.isBasicStarted(), equalTo(status.isBasicStarted()));
-        assertThat(response.getAcknowledgeMessage(), equalTo(acknowledgeMessage));
-        assertThat(ProtocolUtils.equals(response.getAcknowledgeMessages(), ackMessages), equalTo(true));
-    }
-
-    private static void toXContent(StartBasicResponse response, XContentBuilder builder) throws IOException {
-        builder.startObject();
-        builder.field("acknowledged", response.isAcknowledged());
-        if (response.isBasicStarted()) {
-            builder.field("basic_was_started", true);
-        } else {
-            builder.field("basic_was_started", false);
-            builder.field("error_message", response.getErrorMessage());
-        }
-        if (response.getAcknowledgeMessages().isEmpty() == false) {
-            builder.startObject("acknowledge");
-            builder.field("message", response.getAcknowledgeMessage());
-            for (Map.Entry<String, String[]> entry : response.getAcknowledgeMessages().entrySet()) {
-                builder.startArray(entry.getKey());
-                for (String message : entry.getValue()) {
-                    builder.value(message);
-                }
-                builder.endArray();
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-    }
-
-    private static Map<String, String[]> randomAckMessages() {
-        int nFeatures = randomIntBetween(1, 5);
-
-        Map<String, String[]> ackMessages = new HashMap<>();
-
-        for (int i = 0; i < nFeatures; i++) {
-            String feature = randomAlphaOfLengthBetween(9, 15);
-            int nMessages = randomIntBetween(1, 5);
-            String[] messages = new String[nMessages];
-            for (int j = 0; j < nMessages; j++) {
-                messages[j] = randomAlphaOfLengthBetween(10, 30);
-            }
-            ackMessages.put(feature, messages);
-        }
-
-        return ackMessages;
-    }
-
-}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoRequestTests.java
index 2375077220116..86250fdaec274 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/IndexUpgradeInfoRequestTests.java
@@ -23,8 +23,6 @@
 
 public class IndexUpgradeInfoRequestTests extends ESTestCase {
 
-    // TODO: add to cross XPack-HLRC serialization test
-
     public void testNullIndices() {
         expectThrows(NullPointerException.class, () -> new IndexUpgradeInfoRequest((String[])null));
         expectThrows(NullPointerException.class, () -> new IndexUpgradeInfoRequest().indices((String[])null));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterRequestTests.java
new file mode 100644
index 0000000000000..6b39d81f171ca
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterRequestTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.job.config.MlFilter;
+import org.elasticsearch.client.ml.job.config.MlFilterTests;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+
+public class PutFilterRequestTests extends AbstractXContentTestCase<PutFilterRequest> {
+
+    @Override
+    protected PutFilterRequest createTestInstance() {
+        return new PutFilterRequest(MlFilterTests.createRandom());
+    }
+
+    @Override
+    protected PutFilterRequest doParseInstance(XContentParser parser) {
+        return new PutFilterRequest(MlFilter.PARSER.apply(parser, null).build());
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterResponseTests.java
new file mode 100644
index 0000000000000..29eda47598b2f
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutFilterResponseTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.job.config.MlFilterTests;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class PutFilterResponseTests extends AbstractXContentTestCase<PutFilterResponse> {
+
+    @Override
+    protected PutFilterResponse createTestInstance() {
+        return new PutFilterResponse(MlFilterTests.createRandom());
+    }
+
+    @Override
+    protected PutFilterResponse doParseInstance(XContentParser parser) throws IOException {
+        return PutFilterResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobRequestTests.java
new file mode 100644
index 0000000000000..c1271207d41bc
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobRequestTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.rollup;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class DeleteRollupJobRequestTests extends AbstractXContentTestCase<DeleteRollupJobRequest> {
+
+    private String jobId;
+
+    @Before
+    public void setUpOptionalId() {
+        jobId = randomAlphaOfLengthBetween(1, 10);
+    }
+
+    @Override
+    protected DeleteRollupJobRequest createTestInstance() {
+        return new DeleteRollupJobRequest(jobId);
+    }
+
+    @Override
+    protected DeleteRollupJobRequest doParseInstance(final XContentParser parser) throws IOException {
+        return DeleteRollupJobRequest.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testRequireConfiguration() {
+        final NullPointerException e = expectThrows(NullPointerException.class, ()-> new DeleteRollupJobRequest(null));
+        assertEquals("id parameter must not be null", e.getMessage());
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobResponseTests.java
new file mode 100644
index 0000000000000..1dc02ff386de0
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/DeleteRollupJobResponseTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.rollup;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class DeleteRollupJobResponseTests extends AbstractXContentTestCase<DeleteRollupJobResponse> {
+
+    private boolean acknowledged;
+
+    @Before
+    public void setupJobID() {
+        acknowledged = randomBoolean();
+    }
+
+    @Override
+    protected DeleteRollupJobResponse createTestInstance() {
+        return new DeleteRollupJobResponse(acknowledged);
+    }
+
+    @Override
+    protected  DeleteRollupJobResponse doParseInstance(XContentParser parser) throws IOException {
+        return DeleteRollupJobResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/AuthenticateResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/AuthenticateResponseTests.java
new file mode 100644
index 0000000000000..ce813f5ecf59c
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/AuthenticateResponseTests.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.security.user.User;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public class AuthenticateResponseTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        xContentTester(
+                this::createParser,
+                this::createTestInstance,
+                this::toXContent,
+                AuthenticateResponse::fromXContent)
+                .supportsUnknownFields(false)
+                .test();
+    }
+
+    public void testEqualsAndHashCode() {
+        final AuthenticateResponse reponse = createTestInstance();
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(reponse, this::copy,
+            this::mutate);
+    }
+
+    protected AuthenticateResponse createTestInstance() {
+        final String username = randomAlphaOfLengthBetween(1, 4);
+        final List<String> roles = Arrays.asList(generateRandomStringArray(4, 4, false, true));
+        final Map<String, Object> metadata;
+        metadata = new HashMap<>();
+        if (randomBoolean()) {
+            metadata.put("string", null);
+        } else {
+            metadata.put("string", randomAlphaOfLengthBetween(0, 4));
+        }
+        if (randomBoolean()) {
+            metadata.put("string_list", null);
+        } else {
+            metadata.put("string_list", Arrays.asList(generateRandomStringArray(4, 4, false, true)));
+        }
+        final String fullName = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 4));
+        final String email = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 4));
+        final boolean enabled = randomBoolean();
+        return new AuthenticateResponse(new User(username, roles, metadata, fullName, email), enabled);
+    }
+
+    private void toXContent(AuthenticateResponse response, XContentBuilder builder) throws IOException {
+        final User user = response.getUser();
+        final boolean enabled = response.enabled();
+        builder.startObject();
+        builder.field(AuthenticateResponse.USERNAME.getPreferredName(), user.username());
+        builder.field(AuthenticateResponse.ROLES.getPreferredName(), user.roles());
+        builder.field(AuthenticateResponse.METADATA.getPreferredName(), user.metadata());
+        if (user.fullName() != null) {
+            builder.field(AuthenticateResponse.FULL_NAME.getPreferredName(), user.fullName());
+        }
+        if (user.email() != null) {
+            builder.field(AuthenticateResponse.EMAIL.getPreferredName(), user.email());
+        }
+        builder.field(AuthenticateResponse.ENABLED.getPreferredName(), enabled);
+        builder.endObject();
+    }
+
+    private AuthenticateResponse copy(AuthenticateResponse response) {
+        final User originalUser = response.getUser();
+        final User copyUser = new User(originalUser.username(), originalUser.roles(), originalUser.metadata(), originalUser.fullName(),
+                originalUser.email());
+        return new AuthenticateResponse(copyUser, response.enabled());
+    }
+
+    private AuthenticateResponse mutate(AuthenticateResponse response) {
+        final User originalUser = response.getUser();
+        switch (randomIntBetween(1, 6)) {
+            case 1:
+            return new AuthenticateResponse(new User(originalUser.username() + "wrong", originalUser.roles(), originalUser.metadata(),
+                    originalUser.fullName(), originalUser.email()), response.enabled());
+            case 2:
+                final Collection<String> wrongRoles = new ArrayList<>(originalUser.roles());
+                wrongRoles.add(randomAlphaOfLengthBetween(1, 4));
+                return new AuthenticateResponse(new User(originalUser.username(), wrongRoles, originalUser.metadata(),
+                        originalUser.fullName(), originalUser.email()), response.enabled());
+            case 3:
+                final Map<String, Object> wrongMetadata = new HashMap<>(originalUser.metadata());
+                wrongMetadata.put("wrong_string", randomAlphaOfLengthBetween(0, 4));
+                return new AuthenticateResponse(new User(originalUser.username(), originalUser.roles(), wrongMetadata,
+                        originalUser.fullName(), originalUser.email()), response.enabled());
+            case 4:
+                return new AuthenticateResponse(new User(originalUser.username(), originalUser.roles(), originalUser.metadata(),
+                        originalUser.fullName() + "wrong", originalUser.email()), response.enabled());
+            case 5:
+                return new AuthenticateResponse(new User(originalUser.username(), originalUser.roles(), originalUser.metadata(),
+                        originalUser.fullName(), originalUser.email() + "wrong"), response.enabled());
+            case 6:
+                return new AuthenticateResponse(new User(originalUser.username(), originalUser.roles(), originalUser.metadata(),
+                        originalUser.fullName(), originalUser.email()), !response.enabled());
+        }
+        throw new IllegalStateException("Bad random number");
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ClearRealmCacheResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ClearRealmCacheResponseTests.java
new file mode 100644
index 0000000000000..d21ed1a71a0c4
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/ClearRealmCacheResponseTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class ClearRealmCacheResponseTests extends ESTestCase {
+
+    public void testParseFromXContent() throws IOException {
+        final ElasticsearchException exception = new ElasticsearchException("test");
+        final String nodesHeader = "\"_nodes\": { \"total\": 2, \"successful\": 1, \"failed\": 1, \"failures\": [ "
+            + Strings.toString(exception) + "] },";
+        final String clusterName = "\"cluster_name\": \"cn\",";
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{" + nodesHeader + clusterName +  "\"nodes\" : {} }")) {
+
+            ClearRealmCacheResponse response = ClearRealmCacheResponse.fromXContent(parser);
+            assertNotNull(response);
+            assertThat(response.getNodes(), empty());
+            assertThat(response.getClusterName(), equalTo("cn"));
+            assertThat(response.getHeader().getSuccessful(), equalTo(1));
+            assertThat(response.getHeader().getFailed(), equalTo(1));
+            assertThat(response.getHeader().getTotal(), equalTo(2));
+            assertThat(response.getHeader().getFailures(), hasSize(1));
+            assertThat(response.getHeader().getFailures().get(0).getMessage(), containsString("reason=test"));
+        }
+
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            "{" + nodesHeader + clusterName + "\"nodes\" : { " +
+                "\"id1\": { \"name\": \"a\"}, " +
+                "\"id2\": { \"name\": \"b\"}" +
+                "}}")) {
+
+            ClearRealmCacheResponse response = ClearRealmCacheResponse.fromXContent(parser);
+            assertNotNull(response);
+            assertThat(response.getNodes(), hasSize(2));
+            assertThat(response.getNodes().get(0).getId(), equalTo("id1"));
+            assertThat(response.getNodes().get(0).getName(), equalTo("a"));
+            assertThat(response.getNodes().get(1).getId(), equalTo("id2"));
+            assertThat(response.getNodes().get(1).getName(), equalTo("b"));
+        }
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/xpack/XPackInfoResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/xpack/XPackInfoResponseTests.java
new file mode 100644
index 0000000000000..702c4bef64bd2
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/xpack/XPackInfoResponseTests.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.xpack;
+
+import org.elasticsearch.client.license.LicenseStatus;
+import org.elasticsearch.client.xpack.XPackInfoResponse.BuildInfo;
+import org.elasticsearch.client.xpack.XPackInfoResponse.FeatureSetsInfo;
+import org.elasticsearch.client.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet;
+import org.elasticsearch.client.xpack.XPackInfoResponse.LicenseInfo;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+public class XPackInfoResponseTests extends AbstractXContentTestCase<XPackInfoResponse> {
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    protected XPackInfoResponse doParseInstance(XContentParser parser) throws IOException {
+        return XPackInfoResponse.fromXContent(parser);
+    }
+
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        return path -> path.equals("features")
+                || (path.startsWith("features") && path.endsWith("native_code_info"));
+    }
+
+    protected ToXContent.Params getToXContentParams() {
+        Map<String, String> params = new HashMap<>();
+        if (randomBoolean()) {
+            params.put("human", randomBoolean() ? "true" : "false");
+        }
+        if (randomBoolean()) {
+            params.put("categories", "_none");
+        }
+        return new ToXContent.MapParams(params);
+    }
+
+    protected XPackInfoResponse createTestInstance() {
+        return new XPackInfoResponse(
+            randomBoolean() ? null : randomBuildInfo(),
+            randomBoolean() ? null : randomLicenseInfo(),
+            randomBoolean() ? null : randomFeatureSetsInfo());
+    }
+
+    private BuildInfo randomBuildInfo() {
+        return new BuildInfo(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(15));
+    }
+
+    private LicenseInfo randomLicenseInfo() {
+        return new LicenseInfo(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(4),
+            randomAlphaOfLength(5),
+            randomFrom(LicenseStatus.values()),
+            randomLong());
+    }
+
+    private FeatureSetsInfo randomFeatureSetsInfo() {
+        int size = between(0, 10);
+        Set<FeatureSet> featureSets = new HashSet<>(size);
+        while (featureSets.size() < size) {
+            featureSets.add(randomFeatureSet());
+        }
+        return new FeatureSetsInfo(featureSets);
+    }
+
+    private FeatureSet randomFeatureSet() {
+        return new FeatureSet(
+            randomAlphaOfLength(5),
+            randomBoolean() ? null : randomAlphaOfLength(20),
+            randomBoolean(),
+            randomBoolean(),
+            randomNativeCodeInfo());
+    }
+
+    private Map<String, Object> randomNativeCodeInfo() {
+        if (randomBoolean()) {
+            return null;
+        }
+        int size = between(0, 10);
+        Map<String, Object> nativeCodeInfo = new HashMap<>(size);
+        while (nativeCodeInfo.size() < size) {
+            nativeCodeInfo.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        }
+        return nativeCodeInfo;
+    }
+}
diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java
index 90801715b7e20..9191f5025581b 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java
@@ -36,7 +36,7 @@
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory;
+import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
 import org.elasticsearch.client.Node;
 import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.client.Request;
@@ -84,7 +84,8 @@ public class RestClientDocumentation {
         RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
         builder.addHeader("Authorization", "Bearer " + TOKEN); // <1>
         builder.setHttpAsyncResponseConsumerFactory(           // <2>
-            new HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
+            new HttpAsyncResponseConsumerFactory
+                .HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024));
         COMMON_OPTIONS = builder.build();
     }
     // end::rest-client-options-singleton
diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl
index 4ea7e124598ec..cf19a1cd9ddf5 100755
--- a/dev-tools/es_release_notes.pl
+++ b/dev-tools/es_release_notes.pl
@@ -32,7 +32,7 @@
     ">enhancement", ">bug",           ">regression",  ">upgrade"
 );
 my %Ignore = map { $_ => 1 }
-    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build" );
+    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" );
 
 my %Group_Labels = (
     '>breaking'      => 'Breaking changes',
diff --git a/distribution/bwc/maintenance-bugfix-snapshot/build.gradle b/distribution/bwc/bugfix/build.gradle
similarity index 100%
rename from distribution/bwc/maintenance-bugfix-snapshot/build.gradle
rename to distribution/bwc/bugfix/build.gradle
diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle
index f1b2ae01fda84..c466a6ec03da7 100644
--- a/distribution/bwc/build.gradle
+++ b/distribution/bwc/build.gradle
@@ -17,236 +17,225 @@
  * under the License.
  */
 
-
-
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 
 import java.nio.charset.StandardCharsets
 
 import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
+
 /**
- * This is a dummy project which does a local checkout of the previous
- * version's branch, and builds a snapshot. This allows backcompat
- * tests to test against the next unreleased version, closest to this version,
- * without relying on snapshots.
+ *  We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
+ *  For this we need to check out and build the unreleased versions.
+ *  Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
+ *  unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
+ *  and configure them to build various versions here.
  */
-subprojects {
-
-  Version bwcVersion = bwcVersions.getSnapshotForProject(project.name)
-  if (bwcVersion == null) {
-    // this project wont do anything
-    return
-  }
-
-  String bwcBranch
-  if (project.name == 'next-minor-snapshot') {
-    // this is always a .x series
-    bwcBranch = "${bwcVersion.major}.x"
-  } else {
-    bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}"
-  }
-
-  apply plugin: 'distribution'
-  // Not published so no need to assemble
-  assemble.enabled = false
-  assemble.dependsOn.remove('buildBwcVersion')
-
-  File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
-
-  final String remote = System.getProperty("tests.bwc.remote", "elastic")
-
-  final boolean gitFetchLatest
-  final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
-  if ("true".equals(gitFetchLatestProperty)) {
-    gitFetchLatest = true
-  } else if ("false".equals(gitFetchLatestProperty)) {
-    gitFetchLatest = false
-  } else {
-    throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
-  }
-
-  task createClone(type: LoggedExec) {
-    onlyIf { checkoutDir.exists() == false }
-    commandLine = ['git', 'clone', rootDir, checkoutDir]
-  }
-
-  task findRemote(type: LoggedExec) {
-    dependsOn createClone
-    workingDir = checkoutDir
-    commandLine = ['git', 'remote', '-v']
-    ByteArrayOutputStream output = new ByteArrayOutputStream()
-    standardOutput = output
-    doLast {
-      project.ext.remoteExists = false
-      output.toString('UTF-8').eachLine {
-        if (it.contains("${remote}\t")) {
-          project.ext.remoteExists = true
-        }
-      }
-    }
-  }
-
-  task addRemote(type: LoggedExec) {
-    dependsOn findRemote
-    onlyIf { project.ext.remoteExists == false }
-    workingDir = checkoutDir
-    commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
-  }
-
-  task fetchLatest(type: LoggedExec) {
-    onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
-    dependsOn addRemote
-    workingDir = checkoutDir
-    commandLine = ['git', 'fetch', '--all']
-  }
-
-  String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
-  task checkoutBwcBranch(type: LoggedExec) {
-    String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
-    dependsOn fetchLatest
-    workingDir = checkoutDir
-    commandLine = ['git', 'checkout', refspec]
-    doFirst {
-      println "Checking out elasticsearch ${refspec} for branch ${bwcBranch}"
-    }
-  }
-
-  File buildMetadataFile = project.file("build/${project.name}/build_metadata")
-  task writeBuildMetadata(type: LoggedExec) {
-    dependsOn checkoutBwcBranch
-    workingDir = checkoutDir
-    commandLine = ['git', 'rev-parse', 'HEAD']
-    ignoreExitValue = true
-    ByteArrayOutputStream output = new ByteArrayOutputStream()
-    standardOutput = output
-    doLast {
-      if (execResult.exitValue != 0) {
-        output.toString('UTF-8').eachLine { line -> logger.error(line) }
-        execResult.assertNormalExitValue()
-      }
-      project.mkdir(buildMetadataFile.parent)
-      String commit = output.toString('UTF-8')
-      buildMetadataFile.setText("${buildMetadataKey}=${commit}", 'UTF-8')
-      println "Checked out elasticsearch commit ${commit}"
+bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> project("${project.path}:${unreleasedVersion.gradleProjectName}") {
+    Version bwcVersion = unreleasedVersion.version
+    String bwcBranch = unreleasedVersion.branch
+    apply plugin: 'distribution'
+    // Not published so no need to assemble
+    assemble.enabled = false
+    assemble.dependsOn.remove('buildBwcVersion')
+
+    File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
+
+    final String remote = System.getProperty("tests.bwc.remote", "elastic")
+
+    final boolean gitFetchLatest
+    final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
+    if ("true".equals(gitFetchLatestProperty)) {
+        gitFetchLatest = true
+    } else if ("false".equals(gitFetchLatestProperty)) {
+        gitFetchLatest = false
+    } else {
+        throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
     }
-  }
-
-  List<File> artifactFiles = []
-  List<String> projectDirs = []
-  for (String project : ['zip', 'deb', 'rpm']) {
-    String baseDir = "distribution"
-    if (bwcVersion.onOrAfter('6.3.0')) {
-      baseDir += project == 'zip' ? '/archives' : '/packages'
-      // add oss variant first
-      projectDirs.add("${baseDir}/oss-${project}")
-      artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}.${project}"))
+
+    task createClone(type: LoggedExec) {
+        onlyIf { checkoutDir.exists() == false }
+        commandLine = ['git', 'clone', rootDir, checkoutDir]
     }
-    projectDirs.add("${baseDir}/${project}")
-    artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}.${project}"))
-  }
-
-  task buildBwcVersion(type: Exec) {
-    dependsOn checkoutBwcBranch, writeBuildMetadata
-    workingDir = checkoutDir
-    doFirst {
-      // Execution time so that the checkouts are available  
-      List<String> lines = file("$checkoutDir/.ci/java-versions.properties").readLines()
-      environment(
-              'JAVA_HOME',
-              getJavaHome(it, Integer.parseInt(
-                      lines
-                              .findAll({ it.startsWith("ES_BUILD_JAVA=java") })
-                              .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
-                              .join("!!")
-              ))
-      )
-      environment(
-              'RUNTIME_JAVA_HOME',
-              getJavaHome(it, Integer.parseInt(
-                      lines
-                              .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
-                              .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
-                              .join("!!")
-              ))
-      )
+
+    task findRemote(type: LoggedExec) {
+        dependsOn createClone
+        workingDir = checkoutDir
+        commandLine = ['git', 'remote', '-v']
+        ByteArrayOutputStream output = new ByteArrayOutputStream()
+        standardOutput = output
+        doLast {
+            project.ext.remoteExists = false
+            output.toString('UTF-8').eachLine {
+                if (it.contains("${remote}\t")) {
+                    project.ext.remoteExists = true
+                }
+            }
+        }
     }
 
-    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-      executable 'cmd'
-      args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
-    } else {
-      executable new File(checkoutDir, 'gradlew').toString()
+    task addRemote(type: LoggedExec) {
+        dependsOn findRemote
+        onlyIf { project.ext.remoteExists == false }
+        workingDir = checkoutDir
+        commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
     }
-    if (gradle.startParameter.isOffline()) {
-      args "--offline"
+
+    task fetchLatest(type: LoggedExec) {
+        onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
+        dependsOn addRemote
+        workingDir = checkoutDir
+        commandLine = ['git', 'fetch', '--all']
     }
-    for (String dir : projectDirs) {
-      args ":${dir.replace('/', ':')}:assemble"
+
+    String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
+    task checkoutBwcBranch(type: LoggedExec) {
+        String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
+        dependsOn fetchLatest
+        workingDir = checkoutDir
+        commandLine = ['git', 'checkout', refspec]
+        doFirst {
+            println "Checking out elasticsearch ${refspec} for branch ${bwcBranch}"
+        }
     }
-    args "-Dbuild.snapshot=true"
-    final LogLevel logLevel = gradle.startParameter.logLevel
-    if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
-      args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
+
+    File buildMetadataFile = project.file("build/${project.name}/build_metadata")
+    task writeBuildMetadata(type: LoggedExec) {
+        dependsOn checkoutBwcBranch
+        workingDir = checkoutDir
+        commandLine = ['git', 'rev-parse', 'HEAD']
+        ignoreExitValue = true
+        ByteArrayOutputStream output = new ByteArrayOutputStream()
+        standardOutput = output
+        doLast {
+            if (execResult.exitValue != 0) {
+                output.toString('UTF-8').eachLine { line -> logger.error(line) }
+                execResult.assertNormalExitValue()
+            }
+            project.mkdir(buildMetadataFile.parent)
+            String commit = output.toString('UTF-8')
+            buildMetadataFile.setText("${buildMetadataKey}=${commit}", 'UTF-8')
+            println "Checked out elasticsearch commit ${commit}"
+        }
     }
-    final String showStacktraceName = gradle.startParameter.showStacktrace.name()
-    assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
-    if (showStacktraceName.equals("ALWAYS")) {
-      args "--stacktrace"
-    } else if (showStacktraceName.equals("ALWAYS_FULL")) {
-      args "--full-stacktrace"
+
+    List<File> artifactFiles = []
+    List<String> projectDirs = []
+    for (String project : ['zip', 'deb', 'rpm']) {
+        String baseDir = "distribution"
+        if (bwcVersion.onOrAfter('6.3.0')) {
+            baseDir += project == 'zip' ? '/archives' : '/packages'
+            // add oss variant first
+            projectDirs.add("${baseDir}/oss-${project}")
+            artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
+        }
+        projectDirs.add("${baseDir}/${project}")
+        artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
     }
-    standardOutput = new IndentingOutputStream(System.out)
-    errorOutput = new IndentingOutputStream(System.err)
-    doLast {
-      List missing = artifactFiles.grep { file ->
-        false == file.exists()
-      }
-      if (false == missing.empty) {
-        throw new InvalidUserDataException(
-                "Building bwc version didn't generate expected files ${missing}")
-      }
+
+    task buildBwcVersion(type: Exec) {
+        dependsOn checkoutBwcBranch, writeBuildMetadata
+        workingDir = checkoutDir
+        doFirst {
+            // Execution time so that the checkouts are available
+            List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
+            environment(
+                    'JAVA_HOME',
+                    getJavaHome(it, Integer.parseInt(
+                            lines
+                                    .findAll({ it.startsWith("ES_BUILD_JAVA=java") })
+                                    .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
+                                    .join("!!")
+                    ))
+            )
+            environment(
+                    'RUNTIME_JAVA_HOME',
+                    getJavaHome(it, Integer.parseInt(
+                            lines
+                                    .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
+                                    .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
+                                    .join("!!")
+                    ))
+            )
+        }
+
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            executable 'cmd'
+            args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
+        } else {
+            executable new File(checkoutDir, 'gradlew').toString()
+        }
+        if (gradle.startParameter.isOffline()) {
+            args "--offline"
+        }
+        for (String dir : projectDirs) {
+            args ":${dir.replace('/', ':')}:assemble"
+        }
+        args "-Dbuild.snapshot=true"
+        final LogLevel logLevel = gradle.startParameter.logLevel
+        if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
+            args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
+        }
+        final String showStacktraceName = gradle.startParameter.showStacktrace.name()
+        assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
+        if (showStacktraceName.equals("ALWAYS")) {
+            args "--stacktrace"
+        } else if (showStacktraceName.equals("ALWAYS_FULL")) {
+            args "--full-stacktrace"
+        }
+        standardOutput = new IndentingOutputStream(System.out, bwcVersion)
+        errorOutput = new IndentingOutputStream(System.err, bwcVersion)
+        doLast {
+            List missing = artifactFiles.grep { file ->
+                false == file.exists()
+            }
+            if (false == missing.empty) {
+                throw new InvalidUserDataException(
+                        "Building ${bwcVersion} didn't generate expected files ${missing}")
+            }
+        }
     }
-  }
-
-  if (gradle.startParameter.taskNames == ["assemble"]) {
-    // Gradle needs the `artifacts` declaration, including `builtBy` bellow to make projects dependencies on this
-    // project work, but it will also trigger the build of these for the `assemble` task.
-    // Since these are only used for testing, we don't want to assemble them if `assemble` is the single command being
-    // ran.
-    logger.info("Skipping BWC builds since `assemble` is the only task name provided on the command line")
-  } else {
-    artifacts {
-      for (File artifactFile : artifactFiles) {
-        String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
-        String suffix = artifactFile.toString()[-3..-1]
-        'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
-      }
+
+    if (gradle.startParameter.taskNames == ["assemble"]) {
+        // Gradle needs the `artifacts` declaration, including `builtBy` bellow to make projects dependencies on this
+        // project work, but it will also trigger the build of these for the `assemble` task.
+        // Since these are only used for testing, we don't want to assemble them if `assemble` is the single command being
+        // ran.
+        logger.info("Skipping BWC builds since `assemble` is the only task name provided on the command line")
+    } else {
+        artifacts {
+            for (File artifactFile : artifactFiles) {
+                String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
+                String suffix = artifactFile.toString()[-3..-1]
+                'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
+            }
+        }
     }
-  }
-}
+}}
 
 class IndentingOutputStream extends OutputStream {
 
-  public static final byte[] INDENT = "  [bwc] ".getBytes(StandardCharsets.UTF_8)
-  private final OutputStream delegate
+    public final byte[] indent
+    private final OutputStream delegate
 
-  public IndentingOutputStream(OutputStream delegate) {
-      this.delegate = delegate
-  }
+    public IndentingOutputStream(OutputStream delegate, Object version) {
+        this.delegate = delegate
+        indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
+    }
 
-  @Override
-  public void write(int b) {
-    write([b] as int[], 0, 1)
-  }
+    @Override
+    public void write(int b) {
+        write([b] as int[], 0, 1)
+    }
 
-  public void write(int[] bytes, int offset, int length) {
-    for (int i = 0; i < bytes.length; i++) {
-      delegate.write(bytes[i])
-      if (bytes[i] == '\n') {
-        delegate.write(INDENT)
-      }
+    public void write(int[] bytes, int offset, int length) {
+        for (int i = 0; i < bytes.length; i++) {
+            delegate.write(bytes[i])
+            if (bytes[i] == '\n') {
+                delegate.write(indent)
+            }
+        }
     }
-  }
 }
diff --git a/distribution/bwc/next-bugfix-snapshot/build.gradle b/distribution/bwc/maintenance/build.gradle
similarity index 100%
rename from distribution/bwc/next-bugfix-snapshot/build.gradle
rename to distribution/bwc/maintenance/build.gradle
diff --git a/distribution/bwc/next-minor-snapshot/build.gradle b/distribution/bwc/minor/build.gradle
similarity index 100%
rename from distribution/bwc/next-minor-snapshot/build.gradle
rename to distribution/bwc/minor/build.gradle
diff --git a/distribution/bwc/staged-minor-snapshot/build.gradle b/distribution/bwc/staged/build.gradle
similarity index 100%
rename from distribution/bwc/staged-minor-snapshot/build.gradle
rename to distribution/bwc/staged/build.gradle
diff --git a/distribution/src/bin/elasticsearch-service-mgr.exe b/distribution/src/bin/elasticsearch-service-mgr.exe
index 730240403a7da..e5d4b55d91628 100644
Binary files a/distribution/src/bin/elasticsearch-service-mgr.exe and b/distribution/src/bin/elasticsearch-service-mgr.exe differ
diff --git a/distribution/src/bin/elasticsearch-service-x64.exe b/distribution/src/bin/elasticsearch-service-x64.exe
index dab7def75583f..acd94f2507615 100644
Binary files a/distribution/src/bin/elasticsearch-service-x64.exe and b/distribution/src/bin/elasticsearch-service-x64.exe differ
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index 2981bce61899e..92c065fd3d4c7 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,7 +1,7 @@
 :version:               6.6.0
 :major-version:         6.x
-:lucene_version:        7.5.0
-:lucene_version_path:   7_5_0
+:lucene_version:        7.6.0
+:lucene_version_path:   7_6_0
 :branch:                6.x
 :jdk:                   1.8.0_131
 :jdk_major:             8
diff --git a/docs/build.gradle b/docs/build.gradle
index 987ad79e0479b..df42ea6b80a12 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -1078,6 +1078,34 @@ buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['cale
            ]}
 '''
 
+// used by median absolute deviation aggregation
+buildRestTests.setups['reviews'] = '''
+  - do:
+        indices.create:
+          index: reviews
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              _doc:
+                properties:
+                  product:
+                    type: keyword
+                  rating:
+                    type: long
+  - do:
+        bulk:
+          index: reviews
+          type: _doc
+          refresh: true
+          body: |
+            {"index": {"_id": "1"}}
+            {"product": "widget-foo", "rating": 1}
+            {"index": {"_id": "2"}}
+            {"product": "widget-foo", "rating": 5}
+'''
+
 buildRestTests.setups['remote_cluster'] = buildRestTests.setups['host'] + '''
   - do:
       cluster.put_settings:
@@ -1096,3 +1124,33 @@ buildRestTests.setups['remote_cluster_and_leader_index'] = buildRestTests.setups
             index.number_of_shards: 1
             index.soft_deletes.enabled: true
 '''
+
+buildRestTests.setups['seats'] = '''
+  - do:
+        indices.create:
+          index: seats
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              _doc:
+                properties:
+                  theatre:
+                    type: keyword
+                  cost:
+                    type: long
+  - do:
+        bulk:
+          index: seats
+          type: _doc
+          refresh: true
+          body: |
+            {"index":{}}
+            {"theatre": "Skyline", "cost": 1}
+            {"index":{}}
+            {"theatre": "Graye", "cost": 5}
+            {"index":{}}
+            {"theatre": "Graye", "cost": 8}
+            {"index":{}}
+            {"theatre": "Skyline", "cost": 10}'''
\ No newline at end of file
diff --git a/docs/java-rest/high-level/document/term-vectors.asciidoc b/docs/java-rest/high-level/document/term-vectors.asciidoc
index e739e37732fd4..e8d4a25a2cac0 100644
--- a/docs/java-rest/high-level/document/term-vectors.asciidoc
+++ b/docs/java-rest/high-level/document/term-vectors.asciidoc
@@ -62,9 +62,9 @@ include::../execution.asciidoc[]
 
 
 [id="{upid}-{api}-response"]
-==== TermVectorsResponse
+==== Term Vectors Response
 
-The `TermVectorsResponse` contains the following information:
++{response}+ contains the following information:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
@@ -77,7 +77,7 @@ include-tagged::{doc-tests-file}[{api}-response]
 
 
 ===== Inspecting Term Vectors
-If `TermVectorsResponse` contains non-null list of term vectors,
+If +{response}+ contains non-null list of term vectors,
 more information about each term vector can be obtained using the following:
 
 ["source","java",subs="attributes,callouts,macros"]
diff --git a/docs/java-rest/high-level/ml/put-filter.asciidoc b/docs/java-rest/high-level/ml/put-filter.asciidoc
new file mode 100644
index 0000000000000..2582e7715ab59
--- /dev/null
+++ b/docs/java-rest/high-level/ml/put-filter.asciidoc
@@ -0,0 +1,53 @@
+--
+:api: put-filter
+:request: PutFilterRequest
+:response: PutFilterResponse
+--
+[id="{upid}-{api}"]
+=== Put Filter API
+
+The Put Filter API can be used to create a new {ml} filter
+in the cluster. The API accepts a +{request}+ object
+as a request and returns a +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Put Filter Request
+
+A +{request}+ requires the following argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> The configuration of the {ml} filter to create as a `MlFilter`
+
+[id="{upid}-{api}-config"]
+==== Filter Configuration
+
+The `MlFilter` object contains all the details about the {ml} filter
+configuration.
+
+A `MlFilter` contains the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-config]
+--------------------------------------------------
+<1> Required, the filter ID
+<2> Optional, the filter description
+<3> Optional, the items of the filter. A wildcard * can be used at the beginning or the end of an item.
+Up to 10000 items are allowed in each filter.
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ returns the full representation of
+the new {ml} filter if it has been successfully created.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> The newly created `MlFilter`
diff --git a/docs/java-rest/high-level/rollup/delete_job.asciidoc b/docs/java-rest/high-level/rollup/delete_job.asciidoc
new file mode 100644
index 0000000000000..c98a6fb732659
--- /dev/null
+++ b/docs/java-rest/high-level/rollup/delete_job.asciidoc
@@ -0,0 +1,35 @@
+--
+:api: rollup-delete-job
+:request: DeleteRollupJobRequest
+:response: DeleteRollupJobResponse
+--
+
+[id="{upid}-{api}"]
+=== Delete Rollup Job API
+
+
+[id="{upid}-{api}-request"]
+==== Request
+
+The Delete Rollup Job API allows you to delete a job by ID.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> The ID of the job to delete.
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ indicates if the delete command was received.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> Whether or not the delete job request was received.
+
+include::../execution.asciidoc[]
+
+
diff --git a/docs/java-rest/high-level/search/count.asciidoc b/docs/java-rest/high-level/search/count.asciidoc
new file mode 100644
index 0000000000000..f70e1e1fd4d22
--- /dev/null
+++ b/docs/java-rest/high-level/search/count.asciidoc
@@ -0,0 +1,114 @@
+--
+:api: count
+:request: CountRequest
+:response: CountResponse
+--
+[id="{upid}-{api}"]
+
+=== Count API
+
+[id="{upid}-{api}-request"]
+
+==== Count Request
+
+The +{request}+ is used to execute a query and get the number of matches for the query. The query to use in +{request}+ can be
+set in similar way as query in `SearchRequest` using `SearchSourceBuilder`.
+
+In its most basic form, we can add a query to the request:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-basic]
+--------------------------------------------------
+
+<1> Creates the +{request}+. Without arguments this runs against all indices.
+<2> Most search parameters are added to the `SearchSourceBuilder`.
+<3> Add a `match_all` query to the `SearchSourceBuilder`.
+<4> Add the `SearchSourceBuilder` to the +{request}+.
+
+[[java-rest-high-count-request-optional]]
+===== Count Request optional arguments
+
+Let's first look at some of the optional arguments of a +{request}+:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-indices-types]
+--------------------------------------------------
+<1> Restricts the request to an index
+<2> Limits the request to a type
+
+There are a couple of other interesting optional parameters:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-routing]
+--------------------------------------------------
+<1> Set a routing parameter
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-indicesOptions]
+--------------------------------------------------
+<1> Setting `IndicesOptions` controls how unavailable indices are resolved and how wildcard expressions are expanded
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-preference]
+--------------------------------------------------
+<1> Use the preference parameter e.g. to execute the search to prefer local shards. The default is to randomize across shards.
+
+===== Using the SearchSourceBuilder in CountRequest
+
+Both in search and count API calls, most options controlling the search behavior can be set on the `SearchSourceBuilder`,
+which contains more or less the equivalent of the options in the search request body of the Rest API.
+
+Here are a few examples of some common options:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-source-basics]
+--------------------------------------------------
+<1> Create a `SearchSourceBuilder` with default options.
+<2> Set the query. Can be any type of `QueryBuilder`
+
+After this, the `SearchSourceBuilder` only needs to be added to the
++{request}+:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-source-setter]
+--------------------------------------------------
+
+Note subtle difference when using `SearchSourceBuilder` in `SearchRequest` and using `SearchSourceBuilder` in +{request}+ - using
+`SearchSourceBuilder` in `SearchRequest` one can use `SearchSourceBuilder.size()` and `SearchSourceBuilder.from()` methods to set the
+number of search hits to return, and the starting index. In +{request}+ we're interested in total number of matches and these methods
+have no meaning.
+
+The <<java-rest-high-query-builders, Building Queries>> page gives a list of all available search queries with
+their corresponding `QueryBuilder` objects and `QueryBuilders` helper methods.
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== CountResponse
+
+The +{response}+ that is returned by executing the count API call provides total count of hits and details about the count execution
+itself, like the HTTP status code, or whether the request terminated early:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response-1]
+--------------------------------------------------
+
+The response also provides information about the execution on the
+shard level by offering statistics about the total number of shards that were
+affected by the underlying search, and the successful vs. unsuccessful shards. Possible
+failures can also be handled by iterating over an array off
+`ShardSearchFailures` like in the following example:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response-2]
+--------------------------------------------------
+
diff --git a/docs/java-rest/high-level/security/authenticate.asciidoc b/docs/java-rest/high-level/security/authenticate.asciidoc
new file mode 100644
index 0000000000000..e50c64bf9d0f5
--- /dev/null
+++ b/docs/java-rest/high-level/security/authenticate.asciidoc
@@ -0,0 +1,66 @@
+
+--
+:api: authenticate
+:response: AuthenticateResponse
+--
+
+[id="{upid}-{api}"]
+=== Authenticate API
+
+[id="{upid}-{api}-sync"]
+==== Execution
+
+Authenticating and retrieving information about a user can be performed
+using the `security().authenticate()` method:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-execute]
+--------------------------------------------------
+
+This method does not require a request object. The client waits for the
++{response}+ to be returned before continuing with code execution.
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ contains two fields. Firstly, the `user` field
+, accessed with `getUser`, contains all the information about this
+authenticated user. The other field, `enabled`, tells if this user is actually
+usable or has been temporalily deactivated.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> `getUser` retrieves the `User` instance containing the information,
+see {javadoc-client}/security/user/User.html.
+<2> `enabled` tells if this user is usable or is deactivated.
+
+[id="{upid}-{api}-async"]
+==== Asynchronous Execution
+
+This request can also be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-execute-async]
+--------------------------------------------------
+<1> The `ActionListener` to use when the execution completes. This method does
+not require a request object.
+
+The asynchronous method does not block and returns immediately. Once the request
+has completed the `ActionListener` is called back using the `onResponse` method
+if the execution completed successfully or using the `onFailure` method if
+it failed.
+
+A typical listener for a +{response}+ looks like:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-execute-listener]
+--------------------------------------------------
+<1> Called when the execution completed successfully. The response is
+provided as an argument.
+<2> Called in case of a failure. The exception is provided as an argument.
+
diff --git a/docs/java-rest/high-level/security/clear-realm-cache.asciidoc b/docs/java-rest/high-level/security/clear-realm-cache.asciidoc
new file mode 100644
index 0000000000000..5427db148d65e
--- /dev/null
+++ b/docs/java-rest/high-level/security/clear-realm-cache.asciidoc
@@ -0,0 +1,33 @@
+
+--
+:api: clear-realm-cache
+:request: ClearRealmCacheRequest
+:response: ClearRealmCacheResponse
+--
+
+[id="{upid}-{api}"]
+=== Clear Realm Cache API
+
+[id="{upid}-{api}-request"]
+==== Clear Realm Cache Request
+
+A +{request}+ supports defining the name of realms and usernames that the cache should be cleared
+for.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Clear Roles Cache Response
+
+The returned +{response}+ allows to retrieve information about where the cache was cleared.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> the list of nodes that the cache was cleared on
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index e87789c27a812..9d2fd2aaf1f77 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -54,6 +54,7 @@ The Java High Level REST Client supports the following Search APIs:
 * <<{upid}-field-caps>>
 * <<{upid}-rank-eval>>
 * <<{upid}-explain>>
+* <<{upid}-count>>
 
 include::search/search.asciidoc[]
 include::search/scroll.asciidoc[]
@@ -63,6 +64,7 @@ include::search/multi-search-template.asciidoc[]
 include::search/field-caps.asciidoc[]
 include::search/rank-eval.asciidoc[]
 include::search/explain.asciidoc[]
+include::search/count.asciidoc[]
 
 == Miscellaneous APIs
 
@@ -258,6 +260,7 @@ The Java High Level REST Client supports the following Machine Learning APIs:
 * <<{upid}-get-calendars>>
 * <<{upid}-put-calendar>>
 * <<{upid}-delete-calendar>>
+* <<{upid}-put-filter>>
 
 include::ml/put-job.asciidoc[]
 include::ml/get-job.asciidoc[]
@@ -286,6 +289,7 @@ include::ml/get-categories.asciidoc[]
 include::ml/get-calendars.asciidoc[]
 include::ml/put-calendar.asciidoc[]
 include::ml/delete-calendar.asciidoc[]
+include::ml/put-filter.asciidoc[]
 
 == Migration APIs
 
@@ -299,15 +303,16 @@ include::migration/get-assistance.asciidoc[]
 :upid: {mainid}-rollup
 :doc-tests-file: {doc-tests}/RollupDocumentationIT.java
 
-
 The Java High Level REST Client supports the following Rollup APIs:
 
 * <<java-rest-high-x-pack-rollup-put-job>>
+* <<{upid}-rollup-delete-job>>
 * <<{upid}-rollup-start-job>>
 * <<java-rest-high-x-pack-rollup-get-job>>
 * <<{upid}-x-pack-rollup-get-rollup-caps>>
 
 include::rollup/put_job.asciidoc[]
+include::rollup/delete_job.asciidoc[]
 include::rollup/start_job.asciidoc[]
 include::rollup/get_job.asciidoc[]
 include::rollup/get_rollup_caps.asciidoc[]
@@ -325,6 +330,8 @@ The Java High Level REST Client supports the following Security APIs:
 * <<java-rest-high-security-change-password>>
 * <<java-rest-high-security-delete-role>>
 * <<{upid}-clear-roles-cache>>
+* <<{upid}-clear-realm-cache>>
+* <<{upid}-authenticate>>
 * <<java-rest-high-security-get-certificates>>
 * <<java-rest-high-security-put-role-mapping>>
 * <<java-rest-high-security-get-role-mappings>>
@@ -337,6 +344,8 @@ include::security/disable-user.asciidoc[]
 include::security/change-password.asciidoc[]
 include::security/delete-role.asciidoc[]
 include::security/clear-roles-cache.asciidoc[]
+include::security/clear-realm-cache.asciidoc[]
+include::security/authenticate.asciidoc[]
 include::security/get-certificates.asciidoc[]
 include::security/put-role-mapping.asciidoc[]
 include::security/get-role-mappings.asciidoc[]
@@ -384,4 +393,4 @@ don't leak into the rest of the documentation.
 :response!:
 :doc-tests-file!:
 :upid!:
---
\ No newline at end of file
+--
diff --git a/docs/painless/painless-contexts.asciidoc b/docs/painless/painless-contexts.asciidoc
index cc7bc752ec6d9..d3f598525a8b7 100644
--- a/docs/painless/painless-contexts.asciidoc
+++ b/docs/painless/painless-contexts.asciidoc
@@ -44,7 +44,7 @@ specialized code may define new ways to use a Painless script.
                                     | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
 | Metric aggregation reduce         | <<painless-metric-agg-reduce-context, Painless Documentation>>
                                     | {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Elasticsearch Documentation]
-| Bucket aggregation                | <<painless-bucket-agg-context, Painless Documentation>>
+| Bucket script aggregation         | <<painless-bucket-script-agg-context, Painless Documentation>>
                                     | {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Elasticsearch Documentation]
 | Watcher condition                 | <<painless-watcher-condition-context, Painless Documentation>>
                                     | {xpack-ref}/condition-script.html[Elasticsearch Documentation]
diff --git a/docs/painless/painless-contexts/index.asciidoc b/docs/painless/painless-contexts/index.asciidoc
index a71fde0be32a0..df0c6f71e7798 100644
--- a/docs/painless/painless-contexts/index.asciidoc
+++ b/docs/painless/painless-contexts/index.asciidoc
@@ -28,7 +28,7 @@ include::painless-metric-agg-combine-context.asciidoc[]
 
 include::painless-metric-agg-reduce-context.asciidoc[]
 
-include::painless-bucket-agg-context.asciidoc[]
+include::painless-bucket-script-agg-context.asciidoc[]
 
 include::painless-analysis-predicate-context.asciidoc[]
 
diff --git a/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc b/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc
deleted file mode 100644
index 3bb4cae3d3bab..0000000000000
--- a/docs/painless/painless-contexts/painless-bucket-agg-context.asciidoc
+++ /dev/null
@@ -1,21 +0,0 @@
-[[painless-bucket-agg-context]]
-=== Bucket aggregation context
-
-Use a Painless script in an
-{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[bucket aggregation]
-to calculate a value as a result in a bucket.
-
-*Variables*
-
-`params` (`Map`, read-only)::
-        User-defined parameters passed in as part of the query. The parameters
-        include values defined as part of the `buckets_path`.
-
-*Return*
-
-numeric::
-        The calculated value as the result.
-
-*API*
-
-The standard <<painless-api-reference, Painless API>> is available.
\ No newline at end of file
diff --git a/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc b/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc
new file mode 100644
index 0000000000000..5a5306016945d
--- /dev/null
+++ b/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc
@@ -0,0 +1,86 @@
+[[painless-bucket-script-agg-context]]
+=== Bucket script aggregation context
+
+Use a Painless script in an
+{ref}/search-aggregations-pipeline-bucket-script-aggregation.html[`bucket_script` pipeline aggregation]
+to calculate a value as a result in a bucket.
+
+==== Variables
+
+`params` (`Map`, read-only)::
+        User-defined parameters passed in as part of the query. The parameters
+        include values defined as part of the `buckets_path`.
+
+==== Return
+
+numeric::
+        The calculated value as the result.
+
+==== API
+
+The standard <<painless-api-reference, Painless API>> is available.
+
+==== Example
+
+To run this example, first follow the steps in <<painless-context-examples, context examples>>.
+
+The painless context in a `bucket_script` aggregation provides a `params` map.  This map contains both
+user-specified custom values, as well as the values from other aggregations specified in the `buckets_path`
+property.
+
+This example takes the values from a min and max aggregation, calculates the difference,
+and adds the user-specified base_cost to the result:
+
+[source,Painless]
+--------------------------------------------------
+(params.max - params.min) + params.base_cost
+--------------------------------------------------
+
+Note that the values are extracted from the `params` map.  In context, the aggregation looks like this:
+
+[source,js]
+--------------------------------------------------
+GET /seats/_search
+{
+  "size": 0,
+  "aggs": {
+    "theatres": {
+      "terms": {
+        "field": "theatre",
+        "size": 10
+      },
+      "aggs": {
+        "min_cost": {
+          "min": {
+            "field": "cost"
+          }
+        },
+        "max_cost": {
+          "max": {
+            "field": "cost"
+          }
+        },
+        "spread_plus_base": {
+          "bucket_script": {
+            "buckets_path": { <1>
+              "min": "min_cost",
+              "max": "max_cost"
+            },
+            "script": {
+              "params": {
+                "base_cost": 5 <2>
+              },
+              "source": "(params.max - params.min) + params.base_cost"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:seats]
+<1> The `buckets_path` points to two aggregations (`min_cost`, `max_cost`) and adds `min`/`max` variables
+to the `params` map
+<2> The user-specified `base_cost` is also added to the script's `params` map
\ No newline at end of file
diff --git a/docs/painless/painless-contexts/painless-context-examples.asciidoc b/docs/painless/painless-contexts/painless-context-examples.asciidoc
index a6a67e22a5bd9..469f425d1d89f 100644
--- a/docs/painless/painless-contexts/painless-context-examples.asciidoc
+++ b/docs/painless/painless-contexts/painless-context-examples.asciidoc
@@ -46,7 +46,7 @@ the request URL.
 PUT /seats
 {
   "mappings": {
-    "_doc": {
+    "seat": {
       "properties": {
         "theatre":  { "type": "keyword" },
         "play":     { "type": "text"    },
diff --git a/docs/plugins/repository.asciidoc b/docs/plugins/repository.asciidoc
index 9a4e90bebd714..d444170801833 100644
--- a/docs/plugins/repository.asciidoc
+++ b/docs/plugins/repository.asciidoc
@@ -32,7 +32,7 @@ The GCS repository plugin adds support for using Google Cloud Storage service as
 
 The following plugin has been contributed by our community:
 
-* https://github.com/wikimedia/search-repository-swift[Openstack Swift] (by Wikimedia Foundation)
+* https://github.com/BigDataBoutique/elasticsearch-repository-swift[Openstack Swift] (by Wikimedia Foundation and BigData Boutique)
 
 
 include::repository-azure.asciidoc[]
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index 1d185e80f4f96..514528da0d0bd 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -1,12 +1,129 @@
 [[search-aggregations-bucket-datehistogram-aggregation]]
 === Date Histogram Aggregation
 
-A multi-bucket aggregation similar to the <<search-aggregations-bucket-histogram-aggregation,histogram>> except it can
-only be applied on date values. Since dates are represented in Elasticsearch internally as long values, it is possible
-to use the normal `histogram` on dates as well, though accuracy will be compromised. The reason for this is in the fact
-that time based intervals are not fixed (think of leap years and on the number of days in a month). For this reason,
-we need special support for time based data. From a functionality perspective, this histogram supports the same features
-as the normal <<search-aggregations-bucket-histogram-aggregation,histogram>>. The main difference is that the interval can be specified by date/time expressions.
+This multi-bucket aggregation is similar to the normal
+<<search-aggregations-bucket-histogram-aggregation,histogram>>, but it can
+only be used with date values. Because dates are represented internally in 
+Elasticsearch as long values, it is possible, but not as accurate, to use the
+normal `histogram` on dates as well. The main difference in the two APIs is
+that here the interval can be specified using date/time expressions. Time-based
+data requires special support because time-based intervals are not always a
+fixed length.
+
+==== Setting intervals
+
+There seems to be no limit to the creativity we humans apply to setting our
+clocks and calendars. We've invented leap years and leap seconds, standard and
+daylight savings times, and timezone offsets of 30 or 45 minutes rather than a 
+full hour. While these creations help keep us in sync with the cosmos and our
+environment, they can make specifying time intervals accurately a real challenge.
+The only universal truth our researchers have yet to disprove is that a
+millisecond is always the same duration, and a second is always 1000 milliseconds.
+Beyond that, things get complicated.
+
+Generally speaking, when you specify a single time unit, such as 1 hour or 1 day, you
+are working with a _calendar interval_, but multiples, such as 6 hours or 3 days, are
+_fixed-length intervals_.
+
+For example, a specification of 1 day (1d) from now is a calendar interval that
+means "at
+this exact time tomorrow" no matter the length of the day. A change to or from 
+daylight savings time that results in a 23 or 25 hour day is compensated for and the
+specification of "this exact time tomorrow" is maintained. But if you specify 2 or
+more days, each day must be of the same fixed duration (24 hours). In this case, if
+the specified interval includes the change to or from daylight savings time, the
+interval will end an hour sooner or later than you expect.
+
+There are similar differences to consider when you specify single versus multiple
+minutes or hours. Multiple time periods longer than a day are not supported.
+
+Here are the valid time specifications and their meanings:
+
+milliseconds (ms) ::
+Fixed length interval; supports multiples.
+
+seconds (s) ::
+1000 milliseconds; fixed length interval (except for the last second of a
+minute that contains a leap-second, which is 2000ms long); supports multiples.
+
+minutes (m) ::
+All minutes begin at 00 seconds.
+
+* One minute (1m) is the interval between 00 seconds of the first minute and 00
+seconds of the following minute in the specified timezone, compensating for any
+intervening leap seconds, so that the number of minutes and seconds past the 
+hour is the same at the start and end. 
+* Multiple minutes (__n__m) are intervals of exactly 60x1000=60,000 milliseconds
+each.
+
+hours (h) ::
+All hours begin at 00 minutes and 00 seconds.
+
+* One hour (1h) is the interval between 00:00 minutes of the first hour and 00:00
+minutes of the following hour in the specified timezone, compensating for any
+intervening leap seconds, so that the number of minutes and seconds past the hour
+is the same at the start and end. 
+* Multiple hours (__n__h) are intervals of exactly 60x60x1000=3,600,000 milliseconds
+each.
+
+days (d) ::
+All days begin at the earliest possible time, which is usually 00:00:00
+(midnight).
+
+* One day (1d) is the interval between the start of the day and the start of
+of the following day in the specified timezone, compensating for any intervening
+time changes.
+* Multiple days (__n__d) are intervals of exactly 24x60x60x1000=86,400,000
+milliseconds each.
+
+weeks (w) ::
+
+* One week (1w) is the interval between the start day_of_week:hour:minute:second
+and the same day of the week and time of the following week in the specified 
+timezone.
+* Multiple weeks (__n__w) are not supported.
+
+months (M) ::
+
+* One month (1M) is the interval between the start day of the month and time of
+day and the same day of the month and time of the following month in the specified
+timezone, so that the day of the month and time of day are the same at the start
+and end.
+* Multiple months (__n__M) are not supported.
+
+quarters (q) ::
+
+* One quarter (1q) is the interval between the start day of the month and
+time of day and the same day of the month and time of day three months later,
+so that the day of the month and time of day are the same at the start and end. +
+* Multiple quarters (__n__q) are not supported.
+
+years (y) ::
+
+* One year (1y) is the interval between the start day of the month and time of 
+day and the same day of the month and time of day the following year in the 
+specified timezone, so that the date and time are the same at the start and end. +
+* Multiple years (__n__y) are not supported.
+
+NOTE:
+In all cases, when the specified end time does not exist, the actual end time is
+the closest available time after the specified end.
+
+Widely distributed applications must also consider vagaries such as countries that
+start and stop daylight savings time at 12:01 A.M., so end up with one minute of 
+Sunday followed by an additional 59 minutes of Saturday once a year, and countries
+that decide to move across the international date line. Situations like
+that can make irregular timezone offsets seem easy. 
+
+As always, rigorous testing, especially around time-change events, will ensure
+that your time interval specification is
+what you intend it to be.
+
+WARNING:
+To avoid unexpected results, all connected servers and clients must sync to a
+reliable network time service.
+
+==== Examples
 
 Requesting bucket intervals of a month.
 
@@ -27,13 +144,11 @@ POST /sales/_search?size=0
 // CONSOLE
 // TEST[setup:sales]
 
-Available expressions for interval: `year` (`1y`), `quarter` (`1q`), `month` (`1M`), `week` (`1w`),
-`day` (`1d`), `hour` (`1h`), `minute` (`1m`), `second` (`1s`)
-
-Time values can also be specified via abbreviations supported by <<time-units,time units>> parsing.
-Note that fractional time values are not supported, but you can address this by shifting to another
-time unit (e.g., `1.5h` could instead be specified as `90m`). Also note that time intervals larger than
-days do not support arbitrary values but can only be one unit large (e.g. `1y` is valid, `2y` is not).
+You can also specify time values using abbreviations supported by
+<<time-units,time units>> parsing.
+Note that fractional time values are not supported, but you can address this by
+shifting to another
+time unit (e.g., `1.5h` could instead be specified as `90m`). 
 
 [source,js]
 --------------------------------------------------
@@ -52,15 +167,16 @@ POST /sales/_search?size=0
 // CONSOLE
 // TEST[setup:sales]
 
-==== Keys
+===== Keys
 
 Internally, a date is represented as a 64 bit number representing a timestamp
-in milliseconds-since-the-epoch. These timestamps are returned as the bucket
-++key++s. The `key_as_string` is the same timestamp converted to a formatted
-date string using the format specified with the `format` parameter:
+in milliseconds-since-the-epoch (01/01/1970 midnight UTC). These timestamps are
+returned as the ++key++ name of the bucket. The `key_as_string` is the same
+timestamp converted to a formatted
+date string using the `format` parameter sprcification:
 
-TIP: If no `format` is specified, then it will use the first date
-<<mapping-date-format,format>> specified in the field mapping.
+TIP: If you don't specify `format`, the first date
+<<mapping-date-format,format>> specified in the field mapping is used.
 
 [source,js]
 --------------------------------------------------
@@ -113,15 +229,15 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-==== Time Zone
+===== Timezone
 
 Date-times are stored in Elasticsearch in UTC.  By default, all bucketing and
-rounding is also done in UTC. The `time_zone` parameter can be used to indicate
-that bucketing should use a different time zone.
+rounding is also done in UTC. Use the `time_zone` parameter to indicate
+that bucketing should use a different timezone.
 
-Time zones may either be specified as an ISO 8601 UTC offset (e.g. `+01:00` or
-`-08:00`)  or as a timezone id, an identifier used in the TZ database like
-`America/Los_Angeles`.
+You can specify timezones as either an ISO 8601 UTC offset (e.g. `+01:00` or
+`-08:00`)  or as a timezone ID as specified in the IANA timezone database,
+such as`America/Los_Angeles`.
 
 Consider the following example:
 
@@ -151,7 +267,7 @@ GET my_index/_search?size=0
 ---------------------------------
 // CONSOLE
 
-UTC is used if no time zone is specified, which would result in both of these
+If you don't specify a timezone, UTC is used. This would result in both of these
 documents being placed into the same day bucket, which starts at midnight UTC
 on 1 October 2015:
 
@@ -174,8 +290,8 @@ on 1 October 2015:
 ---------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-If a `time_zone` of `-01:00` is specified, then midnight starts at one hour before
-midnight UTC:
+If you specify a `time_zone` of `-01:00`, midnight in that timezone is one hour
+before midnight UTC:
 
 [source,js]
 ---------------------------------
@@ -223,28 +339,27 @@ second document falls into the bucket for 1 October 2015:
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 <1> The `key_as_string` value represents midnight on each day
-    in the specified time zone.
+    in the specified timezone.
 
 WARNING: When using time zones that follow DST (daylight savings time) changes,
 buckets close to the moment when those changes happen can have slightly different
-sizes than would be expected from the used `interval`.
+sizes than you would expect from the used `interval`.
 For example, consider a DST start in the `CET` time zone: on 27 March 2016 at 2am,
-clocks were turned forward 1 hour to 3am local time. When using `day` as `interval`,
+clocks were turned forward 1 hour to 3am local time. If you use `day` as `interval`,
 the bucket covering that day will only hold data for 23 hours instead of the usual
-24 hours for other buckets. The same is true for shorter intervals like e.g. 12h.
-Here, we will have only a 11h bucket on the morning of 27 March when the DST shift
+24 hours for other buckets. The same is true for shorter intervals, like 12h,
+where you'll have only a 11h bucket on the morning of 27 March when the DST shift
 happens.
 
+===== Offset
 
-==== Offset
-
-The `offset` parameter is used to change the start value of each bucket by the
+Use the `offset` parameter to change the start value of each bucket by the
 specified positive (`+`) or negative offset (`-`) duration, such as `1h` for
 an hour, or `1d` for a day. See <<time-units>> for more possible time
 duration options.
 
-For instance, when using an interval of `day`, each bucket runs from midnight
-to midnight.  Setting the `offset` parameter to `+6h` would change each bucket
+For example, when using an interval of `day`, each bucket runs from midnight
+to midnight.  Setting the `offset` parameter to `+6h` changes each bucket
 to run from 6am to 6am:
 
 [source,js]
@@ -301,12 +416,13 @@ documents into buckets starting at 6am:
 -----------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-NOTE: The start `offset` of each bucket is calculated after the `time_zone`
+NOTE: The start `offset` of each bucket is calculated after `time_zone`
 adjustments have been made.
 
-==== Keyed Response
+===== Keyed Response
 
-Setting the `keyed` flag to `true` will associate a unique string key with each bucket and return the ranges as a hash rather than an array:
+Setting the `keyed` flag to `true` associates a unique string key with each
+bucket and returns the ranges as a hash rather than an array:
 
 [source,js]
 --------------------------------------------------
@@ -358,20 +474,25 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-==== Scripts
+===== Scripts
 
-Like with the normal <<search-aggregations-bucket-histogram-aggregation,histogram>>, both document level scripts and
-value level scripts are supported. It is also possible to control the order of the returned buckets using the `order`
-settings and filter the returned buckets based on a `min_doc_count` setting (by default all buckets between the first
-bucket that matches documents and the last one are returned). This histogram also supports the `extended_bounds`
-setting, which enables extending the bounds of the histogram beyond the data itself (to read more on why you'd want to
-do that please refer to the explanation <<search-aggregations-bucket-histogram-aggregation-extended-bounds,here>>).
+As with the normal <<search-aggregations-bucket-histogram-aggregation,histogram>>,
+both document-level scripts and
+value-level scripts are supported. You can control the order of the returned
+buckets using the `order`
+settings and filter the returned buckets based on a `min_doc_count` setting
+(by default all buckets between the first
+bucket that matches documents and the last one are returned). This histogram
+also supports the `extended_bounds`
+setting, which enables extending the bounds of the histogram beyond the data
+itself. For more information, see
+<<search-aggregations-bucket-histogram-aggregation-extended-bounds,`Extended Bounds`>>.
 
-==== Missing value
+===== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how to treat documents that are missing a value.
+By default, they are ignored, but it is also possible to treat them as if they
+have a value.
 
 [source,js]
 --------------------------------------------------
@@ -391,20 +512,22 @@ POST /sales/_search?size=0
 // CONSOLE
 // TEST[setup:sales]
 
-<1> Documents without a value in the `publish_date` field will fall into the same bucket as documents that have the value `2000-01-01`.
+<1> Documents without a value in the `publish_date` field will fall into the
+same bucket as documents that have the value `2000-01-01`.
 
-==== Order
+===== Order
 
-By default the returned buckets are sorted by their `key` ascending, though the order behaviour can be controlled using
-the `order` setting. Supports the same `order` functionality as the <<search-aggregations-bucket-terms-aggregation-order,`Terms Aggregation`>>.
+By default the returned buckets are sorted by their `key` ascending, but you can
+control the order using
+the `order` setting. This setting supports the same `order` functionality as
+<<search-aggregations-bucket-terms-aggregation-order,`Terms Aggregation`>>.
 
 deprecated[6.0.0, Use `_key` instead of `_time` to order buckets by their dates/keys]
 
-==== Use of a script to aggregate by day of the week
+===== Using a script to aggregate by day of the week
 
-There are some cases where date histogram can't help us, like for example, when we need
-to aggregate the results by day of the week.
-In this case to overcome the problem, we can use a script that returns the day of the week:
+When you need to aggregate the results by day of the week, use a script that 
+returns the day of the week:
 
 
 [source,js]
@@ -452,5 +575,5 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-The response will contain all the buckets having as key the relative day of
-the week: 1 for Monday, 2 for Tuesday... 7 for Sunday.
+The response will contain all the buckets having the relative day of
+the week as key : 1 for Monday, 2 for Tuesday... 7 for Sunday.
diff --git a/docs/reference/aggregations/metrics.asciidoc b/docs/reference/aggregations/metrics.asciidoc
index 96597564dac2d..1d3daa2cecae5 100644
--- a/docs/reference/aggregations/metrics.asciidoc
+++ b/docs/reference/aggregations/metrics.asciidoc
@@ -41,6 +41,8 @@ include::metrics/tophits-aggregation.asciidoc[]
 
 include::metrics/valuecount-aggregation.asciidoc[]
 
+include::metrics/median-absolute-deviation-aggregation.asciidoc[]
+
 
 
 
diff --git a/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc b/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc
new file mode 100644
index 0000000000000..2e88b12d92cd1
--- /dev/null
+++ b/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc
@@ -0,0 +1,189 @@
+[[search-aggregations-metrics-median-absolute-deviation-aggregation]]
+=== Median Absolute Deviation Aggregation
+
+This `single-value` aggregation approximates the https://en.wikipedia.org/wiki/Median_absolute_deviation[median absolute deviation]
+of its search results.
+
+Median absolute deviation is a measure of variability. It is a robust
+statistic, meaning that it is useful for describing data that may have
+outliers, or may not be normally distributed. For such data it can be more
+descriptive than standard deviation.
+
+It is calculated as the median of each data point's deviation from the median
+of the entire sample. That is, for a random variable X, the median absolute
+deviation is median(|median(X) - X~i~|).
+
+==== Example
+
+Assume our data represents product reviews on a one to five star scale.
+Such reviews are usually summarized as a mean, which is easily understandable
+but doesn't describe the reviews' variability. Estimating the median absolute
+deviation can provide insight into how much reviews vary from one another.
+
+In this example we have a product which has an average rating of
+3 stars. Let's look at its ratings' median absolute deviation to determine
+how much they vary
+
+[source,js]
+---------------------------------------------------------
+GET reviews/_search
+{
+  "size": 0,
+  "aggs": {
+    "review_average": {
+      "avg": {
+        "field": "rating"
+      }
+    },
+    "review_variability": {
+      "median_absolute_deviation": {
+        "field": "rating" <1>
+      }
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[setup:reviews]
+<1> `rating` must be a numeric field
+
+The resulting median absolute deviation of `2` tells us that there is a fair
+amount of variability in the ratings. Reviewers must have diverse opinions about
+this product.
+
+[source,js]
+---------------------------------------------------------
+{
+  ...
+  "aggregations": {
+    "review_average": {
+      "value": 3.0
+    },
+    "review_variability": {
+      "value": 2.0
+    }
+  }
+}
+---------------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
+==== Approximation
+
+The naive implementation of calculating median absolute deviation stores the
+entire sample in memory, so this aggregation instead calculates an
+approximation. It uses the https://github.com/tdunning/t-digest[TDigest data structure]
+to approximate the sample median and the median of deviations from the sample
+median. For more about the approximation characteristics of TDigests, see
+<<search-aggregations-metrics-percentile-aggregation-approximation>>.
+
+The tradeoff between resource usage and accuracy of a TDigest's quantile
+approximation, and therefore the accuracy of this aggregation's approximation
+of median absolute deviation, is controlled by the `compression` parameter. A
+higher `compression` setting provides a more accurate approximation at the
+cost of higher memory usage. For more about the characteristics of the TDigest
+`compression` parameter see
+<<search-aggregations-metrics-percentile-aggregation-compression>>.
+
+[source,js]
+---------------------------------------------------------
+GET reviews/_search
+{
+  "size": 0,
+  "aggs": {
+    "review_variability": {
+      "median_absolute_deviation": {
+        "field": "rating",
+        "compression": 100
+      }
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[setup:reviews]
+
+The default `compression` value for this aggregation is `1000`. At this
+compression level this aggregation is usually within 5% of the exact result,
+but observed performance will depend on the sample data.
+
+==== Script
+
+This metric aggregation supports scripting. In our example above, product
+reviews are on a scale of one to five. If we wanted to modify them to a scale
+of one to ten, we can using scripting.
+
+To provide an inline script:
+
+[source,js]
+---------------------------------------------------------
+GET reviews/_search
+{
+  "size": 0,
+  "aggs": {
+    "review_variability": {
+      "median_absolute_deviation": {
+        "script": {
+          "lang": "painless",
+          "source": "doc['rating'].value * params.scaleFactor",
+          "params": {
+            "scaleFactor": 2
+          }
+        }
+      }
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[setup:reviews]
+
+To provide a stored script:
+
+[source,js]
+---------------------------------------------------------
+GET reviews/_search
+{
+  "size": 0,
+  "aggs": {
+    "review_variability": {
+      "median_absolute_deviation": {
+        "script": {
+          "id": "my_script",
+          "params": {
+            "field": "rating"
+          }
+        }
+      }
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[setup:reviews,stored_example_script]
+
+==== Missing value
+
+The `missing` parameter defines how documents that are missing a value should be
+treated. By default they will be ignored but it is also possible to treat them
+as if they had a value.
+
+Let's be optimistic and assume some reviewers loved the product so much that
+they forgot to give it a rating. We'll assign them five stars
+
+[source,js]
+---------------------------------------------------------
+GET reviews/_search
+{
+  "size": 0,
+  "aggs": {
+    "review_variability": {
+      "median_absolute_deviation": {
+        "field": "rating",
+        "missing": 5
+      }
+    }
+  }
+}
+---------------------------------------------------------
+// CONSOLE
+// TEST[setup:reviews]
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index c0ee6c56a2fa4..0a71222a3b3f0 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -277,7 +277,7 @@ The number of requests per second effectively executed during the delete by quer
 
 `throttled_until_millis`::
 
-This field should always be equal to zero in a delete by query response. It only
+This field should always be equal to zero in a `_delete_by_query` response. It only
 has meaning when using the <<docs-delete-by-query-task-api, Task API>>, where it
 indicates the next time (in milliseconds since epoch) a throttled request will be
 executed again in order to conform to `requests_per_second`.
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 5e236f05607e4..374256b0ad6cc 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -164,13 +164,7 @@ POST _reindex
 
 `index` and `type` in `source` can both be lists, allowing you to copy from
 lots of sources in one request. This will copy documents from the `_doc` and
-`post` types in the `twitter` and `blog` index. The copied documents would include the
-`post` type in the `twitter` index and the `_doc` type in the `blog` index. For more
-specific parameters, you can use `query`.
-
-The Reindex API makes no effort to handle ID collisions. For such issues, the target index
-will remain valid, but it's not easy to predict which document will survive because
-the iteration order isn't well defined.
+`post` types in the `twitter` and `blog` indices.
 
 [source,js]
 --------------------------------------------------
@@ -181,12 +175,19 @@ POST _reindex
     "type": ["_doc", "post"]
   },
   "dest": {
-    "index": "all_together"
+    "index": "all_together",
+    "type": "_doc"
   }
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT twitter\nPUT blog\n/]
+// TEST[setup:twitter]
+// TEST[s/^/PUT blog\/post\/post1?refresh\n{"test": "foo"}\n/]
+
+NOTE: The Reindex API makes no effort to handle ID collisions so the last
+document written will "win" but the order isn't usually predictable so it is
+not a good idea to rely on this behavior. Instead, make sure that IDs are unique
+using a script.
 
 It's also possible to limit the number of processed documents by setting
 `size`. This will only copy a single document from `twitter` to
@@ -671,7 +672,7 @@ The number of requests per second effectively executed during the reindex.
 
 `throttled_until_millis`::
 
-This field should always be equal to zero in a `_delete_by_query` response. It only
+This field should always be equal to zero in a `_reindex` response. It only
 has meaning when using the <<docs-reindex-task-api, Task API>>, where it
 indicates the next time (in milliseconds since epoch) a throttled request will be
 executed again in order to conform to `requests_per_second`.
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 84af9ed0d86b8..b30ba75c25bf5 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -237,7 +237,7 @@ batch size is `1000`, so if the `requests_per_second` is set to `500`:
 [source,txt]
 --------------------------------------------------
 target_time = 1000 / 500 per second = 2 seconds
-wait_time = target_time - delete_time = 2 seconds - .5 seconds = 1.5 seconds
+wait_time = target_time - write_time = 2 seconds - .5 seconds = 1.5 seconds
 --------------------------------------------------
 
 Since the batch is issued as a single `_bulk` request large batch sizes will
@@ -332,7 +332,7 @@ The number of requests per second effectively executed during the update by quer
 
 `throttled_until_millis`::
 
-This field should always be equal to zero in a delete by query response. It only
+This field should always be equal to zero in an `_update_by_query` response. It only
 has meaning when using the <<docs-update-by-query-task-api, Task API>>, where it
 indicates the next time (in milliseconds since epoch) a throttled request will be
 executed again in order to conform to `requests_per_second`.
diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
new file mode 100644
index 0000000000000..3c8b6c397c07f
--- /dev/null
+++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc
@@ -0,0 +1,89 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-delete-lifecycle]]
+=== Delete Lifecycle Policy API
+++++
+<titleabbrev>Delete Policy</titleabbrev>
+++++
+
+Deletes an existing lifecycle policy
+
+==== Request
+
+`DELETE _ilm/policy/<policy>`
+
+==== Description
+
+Deletes an existing lifecycle policy
+
+==== Path Parameters
+
+`policy` (required)::
+  (string) Identifier for the policy.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  DELETE operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example deletes an existing policy named `my_policy`:
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+DELETE _ilm/policy/my_policy
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc
new file mode 100644
index 0000000000000..95daf0bda1f6b
--- /dev/null
+++ b/docs/reference/ilm/apis/explain.asciidoc
@@ -0,0 +1,296 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-explain]]
+=== Explain Lifecycle API
+++++
+<titleabbrev>Explain Lifecycle</titleabbrev>
+++++
+
+Shows the current lifecycle status for an index.
+
+==== Request
+
+`GET <index>/_ilm/explain`
+
+==== Description
+
+This API returns information relating to the current lifecycle state of an
+index. This includes information such as the currently executing phase, action,
+and step and the timestamp when the index entered them. It also shows the
+definition of the current phase that is being run and in the event that there
+has been a failure, information regarding the failure.
+
+==== Path Parameters
+
+`index` (required)::
+  (string) Identifier for the index.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  GET operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example retrieves the lifecycle state for the index `my_index`:
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_policy",
+    "index.number_of_replicas": 0
+  }
+}
+
+GET /_cluster/health?wait_for_status=green&timeout=10s
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+GET my_index/_ilm/explain
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+When the index is first taken over by ILM you will see a response like the following:
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "my_index": {
+      "index": "my_index",
+      "managed": true, <1>
+      "policy": "my_policy", <2>
+      "lifecycle_date_millis": 1538475653281, <3>
+      "phase": "new", <4>
+      "phase_time_millis": 1538475653317, <5>
+      "action": "complete", <6>
+      "action_time_millis": 1538475653317, <7>
+      "step": "complete", <8>
+      "step_time_millis": 1538475653317 <9>
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[s/"lifecycle_date_millis": 1538475653281/"lifecycle_date_millis": $body.indices.my_index.lifecycle_date_millis/]
+// TESTRESPONSE[s/"phase_time_millis": 1538475653317/"phase_time_millis": $body.indices.my_index.phase_time_millis/]
+// TESTRESPONSE[s/"action_time_millis": 1538475653317/"action_time_millis": $body.indices.my_index.action_time_millis/]
+// TESTRESPONSE[s/"step_time_millis": 1538475653317/"step_time_millis": $body.indices.my_index.step_time_millis/]
+<1> Shows if the index is being managed by ILM. If the index is not managed by
+ILM the other fields will not be shown
+<2> The name of the policy which ILM is using for this index
+<3> The timestamp used for the `min_age`
+<4> The current phase
+<5> The timestamp for when the index entered the current phase
+<6> The current action
+<7> The timestamp for when the index entered the current action
+<8> The current step
+<9> The timestamp for when the index entered the current step
+
+When the policy is running on the index the response will contain a
+`phase_execution` object that describes the exact phase that is being run.
+Changes to the underlying policy will not affect this index until the current
+phase definition has been completely executed.
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "test-000069": {
+      "index": "test-000069",
+      "managed": true,
+      "policy": "my_lifecycle3",
+      "lifecycle_date_millis": 1538475653281,
+      "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "phase": "hot",
+      "phase_time_millis": 1538475653317,
+      "phase_time": "2018-10-15T13:45:22.577Z",
+      "action": "rollover",
+      "action_time_millis": 1538475653317,
+      "action_time": "2018-10-15T13:45:22.577Z",
+      "step": "attempt_rollover",
+      "step_time_millis": 1538475653317,
+      "step_time": "2018-10-15T13:45:22.577Z",
+      "phase_execution": { <1>
+        "policy": "my_lifecycle3", <2>
+        "phase_definition": { <3>
+          "min_age": "0ms",
+          "actions": {
+            "rollover": {
+              "max_age": "30s"
+            }
+          }
+        },
+        "version": 3, <4>
+        "modified_date": "2018-10-15T13:21:41.576Z", <5>
+        "modified_date_in_millis": 1539609701576 <6>
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[skip:not possible to get the cluster into this state in a docs test]
+<1> The phase execution information for this index in its current phase
+<2> The policy that this phase definition was loaded from
+<3> The phase definition itself. This is the JSON for the phase loaded from the
+policy at the time the index entered the current phase
+<4> The version of the policy at the time the phase definition was loaded
+<5> The last modified date of the policy at the time the phase definition was loaded 
+<6> The last modified epoch time of the policy at the time the phase definition was loaded
+
+
+If the policy is waiting for a step to complete for the index, the response will contain step information such as:
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "test-000020": {
+      "index": "test-000020",
+      "managed": true,
+      "policy": "my_lifecycle3",
+      "lifecycle_date_millis": 1538475653281,
+      "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "phase": "warm",
+      "phase_time_millis": 1538475653317,
+      "phase_time": "2018-10-15T13:45:22.577Z",
+      "action": "allocate",
+      "action_time_millis": 1538475653317,
+      "action_time": "2018-10-15T13:45:22.577Z",
+      "step": "check-allocation",
+      "step_time_millis": 1538475653317,
+      "step_time": "2018-10-15T13:45:22.577Z",
+      "step_info": { <1>
+        "message": "Waiting for all shard copies to be active",
+        "shards_left_to_allocate": -1,
+        "all_shards_active": false,
+        "actual_replicas": 2
+      },
+      "phase_execution": {
+        "policy": "my_lifecycle3",
+        "phase_definition": {
+          "min_age": "0ms",
+          "actions": {
+            "allocate": {
+              "number_of_replicas": 2,
+              "include": {
+                "box_type": "warm"
+              },
+              "exclude": {},
+              "require": {}
+            },
+            "forcemerge": {
+              "max_num_segments": 1
+            }
+          }
+        },
+        "version": 2,
+        "modified_date": "2018-10-15T13:20:02.489Z",
+        "modified_date_in_millis": 1539609602489
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[skip:not possible to get the cluster into this state in a docs test]
+<1> `step_info` shows information about what ILM is waiting for on this index.
+In this case we are waiting for all shard copies of the index to be active.
+
+If the index is in the ERROR step, something has gone wrong when executing a
+step in the policy and will need to be investigated and resolved for the index
+to make progress. TO help determine how to resolve the error the explain response 
+will show the step that failed in `failed_step`, and the information on the error
+that occurred in `step_info`.
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "test-000056": {
+      "index": "test-000056",
+      "managed": true,
+      "policy": "my_lifecycle3",
+      "lifecycle_date_millis": 1538475653281,
+      "lifecycle_date": "2018-10-15T13:45:21.981Z",
+      "phase": "hot",
+      "phase_time_millis": 1538475653317,
+      "phase_time": "2018-10-15T13:45:22.577Z",
+      "action": "rollover",
+      "action_time_millis": 1538475653317,
+      "action_time": "2018-10-15T13:45:22.577Z",
+      "step": "ERROR",
+      "step_time_millis": 1538475653317,
+      "step_time": "2018-10-15T13:45:22.577Z",
+      "failed_step": "attempt_rollover", <1>
+      "step_info": { <2>
+        "type": "resource_already_exists_exception",
+        "reason": "index [test-000057/H7lF9n36Rzqa-KfKcnGQMg] already exists",
+        "index_uuid": "H7lF9n36Rzqa-KfKcnGQMg",
+        "index": "test-000057"
+      },
+      "phase_execution": {
+        "policy": "my_lifecycle3",
+        "phase_definition": {
+          "min_age": "0ms",
+          "actions": {
+            "rollover": {
+              "max_age": "30s"
+            }
+          }
+        },
+        "version": 3,
+        "modified_date": "2018-10-15T13:21:41.576Z",
+        "modified_date_in_millis": 1539609701576
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[skip:not possible to get the cluster into this state in a docs test]
+<1> The step that caused an error
+<2> Information on the error that occurred. In this case the next index already
+existed when the rollover operation was performed
diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc
new file mode 100644
index 0000000000000..dbc8a572903b3
--- /dev/null
+++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc
@@ -0,0 +1,115 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-get-lifecycle]]
+=== Get Lifecycle Policy API
+++++
+<titleabbrev>Get Policy</titleabbrev>
+++++
+
+Retrieves an existing policy
+
+==== Request
+
+`GET _ilm/policy`
+`GET _ilm/policy/<policy>`
+
+==== Description
+
+This API returns a policy definition along with some of its metadata like
+its last modified date and version. If no path parameters are provided, then
+all the policies defined will be returned.
+
+==== Path Parameters
+
+`policy` (optional)::
+  (string) Identifier for the policy.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  GET operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example retrieves the policy named `my_policy`:
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+GET _ilm/policy
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "my_policy": {
+    "version": 1, <1>
+    "modified_date": 82392349, <2>
+    "policy": {
+      "phases": {
+        "warm": {
+          "min_age": "10d",
+          "actions": {
+            "forcemerge": {
+              "max_num_segments": 1
+            }
+          }
+        },
+        "delete": {
+          "min_age": "30d",
+          "actions": {
+            "delete": {}
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[s/"modified_date": 82392349/"modified_date": $body.my_policy.modified_date/]
+<1> The version of the policy. This is increased whenever the policy is updated
+<2> The timestamp when this policy was last modified
diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc
new file mode 100644
index 0000000000000..8f5d2289ff2ea
--- /dev/null
+++ b/docs/reference/ilm/apis/get-status.asciidoc
@@ -0,0 +1,55 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-get-status]]
+=== Get ILM Status API
+++++
+<titleabbrev>Get ILM Status</titleabbrev>
+++++
+
+Gets the current status for ILM.
+
+==== Request
+
+`GET /_ilm/status`
+
+==== Description
+
+This API will return the current status of the ILM plugin. The response contains
+a `operation_mode` field which shows whether the ILM plugin is `STARTED`, `STOPPING`
+or `STOPPED`. This `operation_mode` is controlled by the <<ilm-start, Start ILM>>
+and <<ilm-stop, Stop ILM>> APIs.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  get operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example stops the ILM plugin.
+
+[source,js]
+--------------------------------------------------
+GET _ilm/status
+--------------------------------------------------
+// CONSOLE
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "operation_mode": "RUNNING"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/apis/ilm-api.asciidoc b/docs/reference/ilm/apis/ilm-api.asciidoc
new file mode 100644
index 0000000000000..49c7d2155d516
--- /dev/null
+++ b/docs/reference/ilm/apis/ilm-api.asciidoc
@@ -0,0 +1,42 @@
+[[index-lifecycle-management-api]]
+== Index Lifecycle Management API
+
+You can use the following APIs to manage policies on indices.
+
+[float]
+[[ilm-api-policy-endpoint]]
+=== Policy Management APIs
+
+* <<ilm-put-lifecycle,Create Lifecycle Policy>>
+* <<ilm-get-lifecycle,Get Lifecycle Policy>>
+* <<ilm-delete-lifecycle,Delete Lifecycle Policy>>
+
+[float]
+[[ilm-api-index-endpoint]]
+=== Index Management APIs
+
+* <<ilm-move-to-step,Move Index To Step>>
+* <<ilm-retry-policy,Retry Policy On Indices>>
+
+[float]
+[[ilm-api-management-endpoint]]
+=== Operation Management APIs
+
+* <<ilm-get-status,Get ILM Operation Mode>>
+* <<ilm-start,Start ILM>>
+* <<ilm-stop,Stop ILM>>
+* <<ilm-explain,Explain API>>
+
+
+include::put-lifecycle.asciidoc[]
+include::get-lifecycle.asciidoc[]
+include::delete-lifecycle.asciidoc[]
+
+include::move-to-step.asciidoc[]
+include::remove-policy.asciidoc[]
+include::retry-policy.asciidoc[]
+
+include::get-status.asciidoc[]
+include::explain.asciidoc[]
+include::start.asciidoc[]
+include::stop.asciidoc[]
diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc
new file mode 100644
index 0000000000000..c34b800856c10
--- /dev/null
+++ b/docs/reference/ilm/apis/move-to-step.asciidoc
@@ -0,0 +1,121 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-move-to-step]]
+=== Move To Step API
+++++
+<titleabbrev>Move To Step</titleabbrev>
+++++
+
+Moves a managed index into a specific execution step its policy
+
+==== Request
+
+`POST _ilm/move/<index>`
+
+==== Description
+
+WARNING: This is an expert API that may lead to unintended data loss. When used,
+an index's policy will begin executing at the specified step. It will execute
+the step specified even if it has already executed it. Since this is a, potentionally,
+dangerous action, specifying both the current step and next step to move to is
+required in the body of the request.
+
+This API changes the current step for the specified index to the step supplied in the body of the request
+
+==== Path Parameters
+
+`index` (required)::
+  (string) Identifier for the index.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  move operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example moves the index `my_index` from the initial step to the
+forcemerge step:
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_policy"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+POST _ilm/move/my_index
+{
+  "current_step": { <1>
+    "phase": "new",
+    "action": "complete",
+    "name": "complete"
+  },
+  "next_step": { <2>
+    "phase": "warm",
+    "action": "forcemerge",
+    "name": "forcemerge"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+<1> The step that the index is currently expected to be executing
+<2> The step that the index should move to when executing this request
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
+
+NOTE: An error will be returned if the index is now longer executing the step
+specified in `current_step`. This is so the index is not moved from an
+unexpected step into the `next_step`.
diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc
new file mode 100644
index 0000000000000..36650078db652
--- /dev/null
+++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc
@@ -0,0 +1,82 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-put-lifecycle]]
+=== Put Lifecycle Policy API
+++++
+<titleabbrev>Put Policy</titleabbrev>
+++++
+
+Creates or updates an ILM Policy
+
+==== Request
+
+`PUT _ilm/policy/<policy>`
+
+==== Description
+
+This API creates a new Lifecycle Policy, or updates an existing one with the same
+identifier. Each call will replace the existing policy and increment the `version`
+associated with the policy.
+
+NOTE: The `version` is only for informational purposes. Only the latest version
+of the policy is stored.
+
+==== Path Parameters
+
+`policy` (required)::
+  (string) Identifier for the policy.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  PUT operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example creates a new policy named `my_policy`:
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+If the request does not encounter errors, you receive the following result:
+[source,js]
+----
+{
+  "acknowledged": true
+}
+----
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/apis/remove-policy.asciidoc b/docs/reference/ilm/apis/remove-policy.asciidoc
new file mode 100644
index 0000000000000..f37ac9a715d28
--- /dev/null
+++ b/docs/reference/ilm/apis/remove-policy.asciidoc
@@ -0,0 +1,98 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-remove-policy]]
+=== Remove Policy On Index API
+++++
+<titleabbrev>Remove Policy From Index</titleabbrev>
+++++
+
+Unassigns a policy from a specified index pattern
+
+==== Request
+
+`POST <index>/_ilm/remove`
+
+==== Description
+
+This action removes a policy from managing an index. It is effectively the same as setting an index's
+`index.lifecycle.name` setting to null.
+
+==== Path Parameters
+
+`index` (required)::
+  (string) Identifier for the index.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example removes a policy `my_policy` from an index `my_index`.
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_policy"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+POST my_index/_ilm/remove
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "has_failures" : false,
+  "failed_indexes" : []
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc
new file mode 100644
index 0000000000000..7c81f9423ef12
--- /dev/null
+++ b/docs/reference/ilm/apis/retry-policy.asciidoc
@@ -0,0 +1,59 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-retry-policy]]
+=== Retry Policy Execution API
+++++
+<titleabbrev>Retry Policy Execution</titleabbrev>
+++++
+
+Retry executing the policy for an index which has errored.
+
+==== Request
+
+`POST <index>/_ilm/retry`
+
+==== Description
+
+This API will re-run a policy is currently in the ERROR step. It will set the
+policy back to the step where the error occurred and attempt to re-execute it.
+Information on whether an index is in the ERROR step can be obtained from the 
+<<ilm-explain, ILM Explain API>>
+
+==== Path Parameters
+
+`index` (required)::
+  (string) Identifier for the indices to retry in comma-separated format.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  retry operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example retries the policy for index `my_index`.
+
+[source,js]
+--------------------------------------------------
+POST my_index/_ilm/retry
+--------------------------------------------------
+// NOTCONSOLE
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
+// NOTCONSOLE
diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc
new file mode 100644
index 0000000000000..073a584e4d872
--- /dev/null
+++ b/docs/reference/ilm/apis/start.asciidoc
@@ -0,0 +1,90 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-start]]
+=== Start ILM API
+++++
+<titleabbrev>Start ILM</titleabbrev>
+++++
+
+Start the ILM plugin
+
+==== Request
+
+`POST /_ilm/start`
+
+==== Description
+
+This API will start the ILM plugin if it is currently stopped. ILM is started
+by default when the cluster is formed so this API is only needed if ILM has
+been stopped using the <<ilm-stop, Stop ILM API>>.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the completion of the 
+  start operation. When this period of time elapses, the API fails and returns
+  an error. The default value is `30s`. For more information about time units, 
+  see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example starts the ILM plugin.
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+
+POST _ilm/stop
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+POST _ilm/start
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc
new file mode 100644
index 0000000000000..cdc038adabcfc
--- /dev/null
+++ b/docs/reference/ilm/apis/stop.asciidoc
@@ -0,0 +1,101 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-stop]]
+=== Stop ILM API
+++++
+<titleabbrev>Stop ILM</titleabbrev>
+++++
+
+Stop the ILM plugin.
+
+==== Request
+
+`POST /_ilm/stop`
+
+==== Description
+
+This API will stop the ILM plugin. This can be used for period where 
+maintenance is required and ILM should not perform any actions on any indices.
+The API will return as soon as the stop request has been acknowledged but the 
+plugin may not immediately stop but rather need to wait for some operations
+to finish before it's stopped. Progress can be seen using the 
+<<ilm-get-status, Get ILM Status>> API.
+
+==== Request Parameters
+
+`timeout`::
+  (time units) Specifies the period of time to wait for the response. When this 
+  period of time elapses, the API fails and returns an error. The default value
+  is `30s`. For more information about time units, see <<time-units>>.
+
+`master_timeout`::
+  (time units) Specifies the period of time to wait for the connection with master.
+  When this period of time elapses, the API fails and returns an error.
+  The default value is `30s`. For more information about time units, see <<time-units>>.
+
+
+==== Examples
+
+The following example stops the ILM plugin.
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+POST _ilm/stop
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged": true
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+POST _ilm/start
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+//////////////////////////
diff --git a/docs/reference/ilm/get-index-lifecycle-information.asciidoc b/docs/reference/ilm/get-index-lifecycle-information.asciidoc
new file mode 100644
index 0000000000000..3d5dc8a172010
--- /dev/null
+++ b/docs/reference/ilm/get-index-lifecycle-information.asciidoc
@@ -0,0 +1,11 @@
+[role="xpack"]
+[[get-index-lifecycle-information]]
+== Get index lifecycle information
+
+Execution Model
+Discuss how actions are actually split up into discrete steps and how you can see more information about where an index is within a policy (info and all)
+Talk about the jump-to-step API
+Error Handling
+Show error in explain api
+Demonstrate the retry API
+Show how to get a sense of progress for things like the allocate step
diff --git a/docs/reference/ilm/getting-started-ilm.asciidoc b/docs/reference/ilm/getting-started-ilm.asciidoc
new file mode 100644
index 0000000000000..ad3596e92a1ad
--- /dev/null
+++ b/docs/reference/ilm/getting-started-ilm.asciidoc
@@ -0,0 +1,15 @@
+[role="xpack"]
+[[getting-started-index-lifecycle-management]]
+== Getting started with {ilm}
+
+Create a policy that rolls over after 1 day deletes an index after 30 days
+
+Show create policy API req/res
+
+Show assign policy to index API req/res
+
+Show both the API and how it is done with `index.lifecyce.name` using the
+create-index API
+
+Show explain API to show current state, but ignore the “step” related info,
+only focus on managed/phase/action
diff --git a/docs/reference/ilm/index.asciidoc b/docs/reference/ilm/index.asciidoc
new file mode 100644
index 0000000000000..d85f92fb1c28a
--- /dev/null
+++ b/docs/reference/ilm/index.asciidoc
@@ -0,0 +1,62 @@
+[role="xpack"]
+[testenv="basic"]
+[[index-lifecycle-management]]
+= Managing Indices
+
+:ilm:             index lifecycle management
+:ILM:             Index lifecycle management
+[partintro]
+--
+The <<index-lifecycle-management-api, {ilm} APIs>> enable you to automate how you
+want to manage your indices over time. Rather than simply performing management
+actions on your indices on a set schedule, you can base actions on other factors
+such as shard size and performance requirements.
+
+You control how indices are handled as they age by attaching a
+lifecycle policy to the index template used to create them. You can update
+the policy to modify the lifecycle of both new and existing indices.
+
+For time series indices, there are four stages in the index lifecycle:
+
+* Hot--the index is actively being updated and queried.
+* Warm--the index is no longer being updated, but is still being queried.
+* Cold--the index is no longer being updated and is seldom queried. The
+information still needs to be searchable, but it's okay if those queries are
+slower.
+* Delete--the index is no longer needed and can safely be deleted.
+
+The lifecycle policy governs how the index transitions through these stages and
+the actions that are performed on the index at each stage. The policy can
+specify:
+
+* The maximum size or age at which you want to roll over to a new index.
+* The point at which the index is no longer being updated and the number of
+primary shards can be reduced.
+* When to force a merge to permanently delete documents marked for deletion.
+* The point at which the index can be moved to less performant hardware.
+* The point at which the availability is not as critical and the number of
+replicas can be reduced.
+* When the index can be safely deleted.
+
+For example, if you are indexing metrics data from a fleet of ATMs into
+Elasticsearch, you might define a policy that says:
+
+. When the index reaches 5GB, roll over to a new index.
+. Move the old index into the warm stage, mark it read only, and shrink it down
+to a single shard.
+. After 7 days, move the index into the cold stage and move it to less expensive
+hardware.
+. Delete the index once the required 30 day retention period is reached.
+--
+
+include::getting-started-ilm.asciidoc[]
+
+include::using-policies-rollover.asciidoc[]
+
+include::set-up-lifecycle-policy.asciidoc[]
+
+include::update-lifecycle-policy.asciidoc[]
+
+include::get-index-lifecycle-information.asciidoc[]
+
+include::start-stop-ilm.asciidoc[]
diff --git a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc
new file mode 100644
index 0000000000000..7f5bb84c598a4
--- /dev/null
+++ b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc
@@ -0,0 +1,112 @@
+[role="xpack"]
+[testenv="basic"]
+[[set-up-lifecycle-policy]]
+== Set up {ilm} policy
+
+In order for an index to use an {ilm} policy to manage its lifecycle we must
+first define a lifecycle policy for it to use. The following request creates
+a policy called `my_policy` in Elasticsearch which we can later use to manage
+our indexes.
+
+[source,js]
+------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "25GB" <1>
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {} <2>
+        }
+      }
+    }
+  }
+}
+------------------------ 
+// CONSOLE
+<1> Rollover the index when it reaches 25GB in size
+<2> Delete the index when its 30 days old
+
+{ilm} will manage an index using the policy defined in the 
+`index.lifecycle.name` index setting. If this setting does not exist in the
+settings for a particular index {ilm} will not manage that index.
+
+To set the policy for an index there are two options:
+1. Apply the policy to an index template and bootstrap creating the first index
+2. Apply the policy to a new index in a create index request
+
+=== Applying a policy to an index template
+
+The `index.lifecycle.name` setting can be set in an index template so that it
+is automatically applied to indexes matching the templates index pattern:
+
+[source,js]
+-----------------------
+PUT _template/my_template
+{
+  "index_patterns": ["test-*"], <1>
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 1,
+    "index.lifecycle.name": "my_policy", <2>
+    "index.lifecycle.rollover_alias": "test-alias"
+  }
+}
+-----------------------
+// CONSOLE
+<1> This template will be applied to all indexes which have a name starting
+with `test-`
+<2> The template will set the policy to be used to `my_policy`
+
+Now that a policy exists and is used in an index template we can create an
+initial index which will be managed by our policy:
+
+[source,js]
+-----------------------
+PUT test-000001
+{
+  "aliases": {
+    "test-alias":{
+      "is_write_index": true <1>
+    }
+  }
+}
+-----------------------
+// CONSOLE
+<1> Set this initial index to be the write index for this alias.
+
+We can now write data to the `test-alias` alias. Because we have a rollover
+action defined in our policy when the index grows larger than 25GB {ilm} will
+create a new index and roll the alias over to use the new index automatically.
+
+=== Apply a policy to a create index request
+
+The `index.lifecycle.name` setting can be set on an individual create index
+request so {ilm} immediately starts managing the index:
+
+[source,js]
+-----------------------
+PUT test-index
+{
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 1,
+    "index.lifecycle.name": "my_policy"
+  }
+}
+-----------------------
+// CONSOLE
+
+IMPORTANT: Its recommended not to use the create index API with a policy that
+defines a rollover action. If you do so, the new index as the result of the
+rollover will not carry forward the policy. Always use index templates to
+define policies with rollover actions.
+
diff --git a/docs/reference/ilm/start-stop-ilm.asciidoc b/docs/reference/ilm/start-stop-ilm.asciidoc
new file mode 100644
index 0000000000000..938b97d44721f
--- /dev/null
+++ b/docs/reference/ilm/start-stop-ilm.asciidoc
@@ -0,0 +1,168 @@
+[role="xpack"]
+[testenv="basic"]
+[[start-stop-ilm]]
+== Start And Stop {ilm}
+
+All indices that are managed by ILM will continue to execute
+their policies. There may be times when this is not desired on certain
+indices, or maybe even all the indices in a cluster. For example,
+maybe there are scheduled maintenance windows when cluster topology
+changes are desired that may impact running ILM actions. For this reason,
+ILM has two ways to disable operations.
+
+Normally, ILM will be running by default.
+To see the current operating status of ILM, use the <<ilm-get-status,Get Status API>>
+to see the current state of ILM.
+
+////
+[source,js]
+--------------------------------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "10d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_policy"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+////
+
+[source,js]
+--------------------------------------------------
+GET _ilm/status
+--------------------------------------------------
+// CONSOLE
+
+If the request does not encounter errors, you receive the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "operation_mode": "RUNNING"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
+
+The operating modes of ILM:
+
+
+.ILM Operating Modes
+|===
+|Name |Description
+|RUNNING |Normal operation where all policies are executed as normal
+|STOPPING|ILM has received a request to stop but is still processing some policies
+|STOPPED |This represents a state where no policies are executed
+|===
+
+=== Stopping ILM
+
+The ILM service can be paused such that no further steps will be executed
+using the <<ilm-stop,Stop API>>.
+
+[source,js]
+--------------------------------------------------
+POST _ilm/stop
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+When stopped, all further policy actions will be halted. This will
+be reflected in the Status API
+
+////
+[source,js]
+--------------------------------------------------
+GET _ilm/status
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+[source,js]
+--------------------------------------------------
+{
+  "operation_mode": "STOPPING"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
+
+The ILM service will then, asynchronously, run all policies to a point
+where it is safe to stop. After ILM verifies that it is safe, it will
+move to the `STOPPED` mode.
+
+////
+[source,js]
+--------------------------------------------------
+PUT trigger_ilm_cs_action
+
+GET _ilm/status
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+[source,js]
+--------------------------------------------------
+{
+  "operation_mode": "STOPPED"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
+
+=== Starting ILM
+
+To start ILM and continue executing policies, use the <<ilm-start, Start API>>.
+
+
+[source,js]
+--------------------------------------------------
+POST _ilm/start
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+////
+[source,js]
+--------------------------------------------------
+GET _ilm/status
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+The Start API will send a request to the ILM service to immediately begin
+normal operations.
+
+[source,js]
+--------------------------------------------------
+{
+  "operation_mode": "RUNNING"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/ilm/update-lifecycle-policy.asciidoc b/docs/reference/ilm/update-lifecycle-policy.asciidoc
new file mode 100644
index 0000000000000..334b5a953fd0e
--- /dev/null
+++ b/docs/reference/ilm/update-lifecycle-policy.asciidoc
@@ -0,0 +1,432 @@
+[role="xpack"]
+[testenv="basic"]
+[[update-lifecycle-policy]]
+== Update lifecycle policy
+
+Updating existing ILM policies is useful to fix mistakes or change
+strategies for newly created indices. It is possible to update policy definitions
+and an index's `index.lifecycle.name` settings independently. To prevent the situation
+that phase definitions are modified while currently being executed on an index, each index
+will keep the version of the current phase definition it began execution with until it completes.
+
+There are three scenarios for examining the behavior updating policies and
+their effects on policy execution on indices.
+
+=== Updates to policies not managing indices
+
+Indices not referencing an existing policy that is updated will not be affected.
+If an index is assigned to the policy, it will be assigned the latest version of that policy
+
+To show this, let's create a policy `my_policy`.
+
+[source,js]
+------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "25GB"
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+------------------------
+// CONSOLE
+
+This newly defined policy will be created and assigned to have a version equal
+to 1. Since we haven't assigned any indices to this policy, any updates that
+occur will be reflected completely on indices that are newly set to be managed
+by this policy.
+
+Updating the Delete phase's minimum age can be done in an update request.
+
+[source,js]
+------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "25GB"
+          }
+        }
+      },
+      "delete": {
+        "min_age": "10d", <1>
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+------------------------
+// CONSOLE
+// TEST[continued]
+<1> update `min_age` to 10 days
+
+//////////
+[source,js]
+--------------------------------------------------
+GET _ilm/policy
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+//////////
+
+When we get the policy, we will see it reflect our latest changes, but
+with its version bumped to 2.
+
+[source,js]
+--------------------------------------------------
+{
+  "my_policy": {
+    "version": 2, <1>
+    "modified_date": 82392349, <2>
+    "policy": {
+      "phases": {
+        "hot": {
+          "min_age": "0ms",
+          "actions": {
+            "rollover": {
+              "max_size": "25gb"
+            }
+          }
+        },
+        "delete": {
+          "min_age": "10d",
+          "actions": {
+            "delete": {}
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[s/"modified_date": 82392349/"modified_date": $body.my_policy.modified_date/]
+<1> The updated version value
+<2> The timestamp when this policy was updated last.
+
+Afterwords, any indices set to `my_policy` will execute against version 2 of
+the policy.
+
+=== Updates to executing policies
+
+Indices preserve the phase definition from the latest policy version that existed
+at the time that it entered that phase. Changes to the currently-executing phase within policy updates will
+not be reflected during execution. This means that updates to the `hot` phase, for example, will not affect
+indices that are currently executing the corresponding `hot` phase.
+
+Let's say we have an index `my_index` managed by the below `my_executing_policy` definition.
+
+[source,js]
+------------------------
+PUT _ilm/policy/my_executing_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_docs": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "10d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+------------------------
+// CONSOLE
+
+////
+[source,js]
+------------------------
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_executing_policy"
+  }
+}
+------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+The <<ilm-explain,Explain API>> is useful to introspect managed indices to see which phase definition they are currently executing.
+Using this API, we can find out that `my_index` is currently attempting to be rolled over.
+
+[source,js]
+--------------------------------------------------
+GET my_index/_ilm/explain
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "my_index": {
+      "index": "my_index",
+      "managed": true,
+      "policy": "my_executing_policy",
+      "lifecycle_date_millis": 1538475653281,
+      "phase": "hot",
+      "phase_time_millis": 1538475653317,
+      "action": "rollover",
+      "action_time_millis": 1538475653317,
+      "step": "attempt_rollover",
+      "step_time_millis": 1538475653317,
+      "phase_execution": {
+        "policy": "my_executing_policy",
+        "modified_date_in_millis": 1538475653317,
+        "version": 1,
+        "phase_definition": {
+          "min_age": "0ms",
+          "actions": {
+            "rollover": {
+              "max_docs": 1
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[s/"lifecycle_date_millis": 1538475653281/"lifecycle_date_millis": $body.indices.my_index.lifecycle_date_millis/]
+// TESTRESPONSE[s/"phase_time_millis": 1538475653317/"phase_time_millis": $body.indices.my_index.phase_time_millis/]
+// TESTRESPONSE[s/"action_time_millis": 1538475653317/"action_time_millis": $body.indices.my_index.action_time_millis/]
+// TESTRESPONSE[s/"step_time_millis": 1538475653317/"step_time_millis": $body.indices.my_index.step_time_millis/]
+// TESTRESPONSE[s/"modified_date_in_millis": 1538475653317/"modified_date_in_millis": $body.indices.my_index.phase_execution.modified_date_in_millis/]
+
+Updating `my_executing_policy` to have no rollover action and, instead, go directly into a newly introduced `warm` phase.
+
+[source,js]
+------------------------
+PUT _ilm/policy/my_executing_policy
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "min_age": "1d",
+        "actions": {
+          "forcemerge": {
+            "max_num_segments": 1
+          }
+        }
+      },
+      "delete": {
+        "min_age": "10d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+------------------------
+// CONSOLE
+// TEST[continued]
+
+Now, version 2 of this policy has no `hot` phase, but if we run the Explain API again, we will see that nothing has changed.
+The index `my_index` is still executing version 1 of the policy.
+
+////
+[source,js]
+--------------------------------------------------
+GET my_index/_ilm/explain
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "my_index": {
+      "index": "my_index",
+      "managed": true,
+      "policy": "my_executing_policy",
+      "lifecycle_date_millis": 1538475653281,
+      "phase": "hot",
+      "phase_time_millis": 1538475653317,
+      "action": "rollover",
+      "action_time_millis": 1538475653317,
+      "step": "attempt_rollover",
+      "step_time_millis": 1538475653317,
+      "phase_execution": {
+        "policy": "my_executing_policy",
+        "modified_date_in_millis": 1538475653317,
+        "version": 1,
+        "phase_definition": {
+          "min_age": "0ms",
+          "actions": {
+            "rollover": {
+              "max_docs": 1
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[s/"lifecycle_date_millis": 1538475653281/"lifecycle_date_millis": $body.indices.my_index.lifecycle_date_millis/]
+// TESTRESPONSE[s/"phase_time_millis": 1538475653317/"phase_time_millis": $body.indices.my_index.phase_time_millis/]
+// TESTRESPONSE[s/"action_time_millis": 1538475653317/"action_time_millis": $body.indices.my_index.action_time_millis/]
+// TESTRESPONSE[s/"step_time_millis": 1538475653317/"step_time_millis": $body.indices.my_index.step_time_millis/]
+// TESTRESPONSE[s/"modified_date_in_millis": 1538475653317/"modified_date_in_millis": $body.indices.my_index.phase_execution.modified_date_in_millis/]
+
+After indexing one document into `my_index` so that rollover succeeds and moves onto the next phase, we will notice something new. The
+index will move into the next phase in the updated version 2 of its policy.
+
+////
+[source,js]
+--------------------------------------------------
+PUT my_index/_doc/1
+{
+  "foo": "bar"
+}
+
+GET my_index/_ilm/explain
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+////
+
+[source,js]
+--------------------------------------------------
+{
+  "indices": {
+    "my_index": {
+      "index": "my_index",
+      "managed": true,
+      "policy": "my_executing_policy",
+      "lifecycle_date_millis": 1538475653281,
+      "phase": "warm",
+      "phase_time_millis": 1538475653317,
+      "action": "forcemerge",
+      "action_time_millis": 1538475653317,
+      "step": "forcemerge",
+      "step_time_millis": 1538475653317,
+      "phase_execution": {
+        "policy": "my_executing_policy",
+        "modified_date_in_millis": 1538475653317,
+        "version": 2, <1>
+        "phase_definition": {
+          "min_age": "1d",
+          "actions": {
+            "forcemerge": {
+              "max_num_segments": 1
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TESTRESPONSE[skip:There is no way to force the index to move to the next step in a timely manner]
+<1> The index has moved to using version 2 of the policy
+
+`my_index` will move to the next phase in the latest policy definition, which is the newly added `warm` phase.
+
+=== Switching policies for an index
+
+Setting `index.lifecycle.name` to a different policy behaves much like a policy update, but instead of just
+switching to a different version, it switches to a different policy.
+
+After setting a policy for an index, we can switch out `my_policy` with
+`my_other_policy` by just updating the index's `index.lifecycle.name`
+setting to the new policy. After completing its currently executed phase,
+it will move on to the next phase in `my_other_policy`. So if it was on the
+`hot` phase before, it will move to the `delete` phase after the `hot` phase concluded.
+
+////
+[source,js]
+------------------------
+PUT _ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "25GB"
+          }
+        }
+      },
+      "delete": {
+        "min_age": "10d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT _ilm/policy/my_other_policy
+{
+  "policy": {
+    "phases": {
+      "delete": {
+        "min_age": "1d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+
+PUT my_index
+{
+  "settings": {
+    "index.lifecycle.name": "my_policy"
+  }
+}
+------------------------
+// CONSOLE
+
+////
+
+[source,js]
+--------------------------------------------------
+PUT my_index/_settings
+{
+  "lifecycle.name": "my_other_policy"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The change to the new policy will not happen immediately. The currently executing phase
+of the existing policy for `my_index` will continue to execute until it completes. Once
+completed, `my_index` will move to being managed by the `my_other_policy`.
diff --git a/docs/reference/ilm/using-policies-rollover.asciidoc b/docs/reference/ilm/using-policies-rollover.asciidoc
new file mode 100644
index 0000000000000..f7982af4fec81
--- /dev/null
+++ b/docs/reference/ilm/using-policies-rollover.asciidoc
@@ -0,0 +1,117 @@
+[role="xpack"]
+[testenv="basic"]
+[[using-policies-rollover]]
+== Using policies to manage index rollover
+
+The rollover action enables you to automatically roll over to a new index based
+on the index size, document count, or age. When a rollover is triggered, a new
+index is created, the write alias is updated to point to the new index, and all
+subsequent updates are written to the new index.
+
+Rolling over to a new index based on size, document count, or age is preferable
+to time-based rollovers. Rolling over at an arbitrary time often results in
+many small indices, which can have a negative impact on performance and
+resource usage.
+
+You control when the rollover action is triggered by specifying one or more
+rollover parameters. The rollover is performed once any of the criteria are
+met. Because the criteria are checked periodically, the index might grow
+slightly beyond the specified threshold. To control how often the critera are
+checked, specify the `indices.lifecycle.poll_interval` cluster setting.
+
+The rollover action takes the following parameters:
+
+.`rollover` Action Parameters
+|===
+|Name |Description
+|max_size |The maximum estimated size the index is allowed to grow
+to. Defaults tonull. Optional.
+|max_docs |The maximum number of document the index should
+contain. Defaults tonull. Optional.
+|max_age |The maximum age of the index. Defaults to `null`. Optional.
+|===
+
+These parameters are used to determine when the index is considered "full" and
+a rollover should be performed. Where multiple criteria are defined the
+rollover operation will be performed once any of the criteria are met.
+
+The following request defines a policy with a rollover action that triggers
+when the index size reaches 25GB. The old index is subsequently deleted after
+30 days.
+
+NOTE: Once an index rolls over, {ilm} uses the timestamp of the rollover
+operation rather than the index creation time to evaluate when to move the
+index to the next phase. For indices that have rolled over, the `min_age`
+criteria specified for a phase is relative to the rollover time for indices. In
+this example, that means the index will be deleted 30 days after rollover, not
+30 days from when the index was created.
+
+[source,js]
+--------------------------------------------------
+PUT /_ilm/policy/my_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "25GB"
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+To use an {ilm} policy, you need to specify it in the index template used to
+create the indices. For example, the following template associates `my_policy`
+with indices created from the template `my_template`.
+
+[source,js]
+-----------------------
+PUT _template/my_template
+{
+  "index_patterns": ["test-*"], <1>
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 1,
+    "index.lifecycle.name": "my_policy", <2>
+    "index.lifecycle.rollover_alias": "test-alias" <3>
+  }
+}
+-----------------------
+// CONSOLE
+<1> Template applies to all indices with the prefix test-
+<2> Associates my_policy with all indices created with this template
+<3> Rolls over the write alias test when the rollover action is triggered
+
+To be able to start using the policy for these `test-*` indexes we need to
+bootstrap the process by creating the first index.
+
+[source,js]
+-----------------------
+PUT test-000001 <1>
+{
+  "aliases": {
+    "test-alias":{
+      "is_write_index": true <2>
+    }
+  }
+}
+-----------------------
+// CONSOLE
+<1> Creates the index called test-000001. The rollover action increments the
+suffix number for each subsequent index.
+<2> Designates this index as the write index for this alias.
+
+When the rollover is performed, the newly-created index is set as the write
+index for the rolled over alias. Documents sent to the alias are indexed into
+the new index, enabling indexing to continue uninterrupted.
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 2905688f7be78..6f80d95079e26 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -57,7 +57,7 @@ include::index-modules.asciidoc[]
 
 include::ingest.asciidoc[]
 
-include::ccr/index.asciidoc[]
+include::ilm/index.asciidoc[]
 
 include::sql/index.asciidoc[]
 
diff --git a/docs/reference/ingest/ingest-node-common-processor.asciidoc b/docs/reference/ingest/ingest-node-common-processor.asciidoc
new file mode 100644
index 0000000000000..dcf8b63630b4b
--- /dev/null
+++ b/docs/reference/ingest/ingest-node-common-processor.asciidoc
@@ -0,0 +1,5 @@
+| `if`             | no       | -        | Conditionally execute this processor.
+| `on_failure`     | no       | -        | Handle failures for this processor. See <<handling-failure-in-pipelines>>.
+| `ignore_failure` | no       | `false`  | Ignore failures for this processor. See <<handling-failure-in-pipelines>>.
+| `tag`            | no       | -        | An identifier for this processor. Useful for debugging and metrics.
+// TODO: See <<ingest-conditionals>>. <-- for the if description once PR 35044 is merged
\ No newline at end of file
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 2a26b0e5d4279..2e14028b8cf01 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -764,6 +764,12 @@ A node will not start if either of these plugins are not available.
 The <<ingest-stats,node stats API>> can be used to fetch ingest usage statistics, globally and on a per
 pipeline basis. Useful to find out which pipelines are used the most or spent the most time on preprocessing.
 
+[float]
+=== Ingest Processor Plugins
+
+Additional ingest processors can be implemented and installed as Elasticsearch {plugins}/intro.html[plugins].
+See {plugins}/ingest.html[Ingest plugins] for information about the available ingest plugins.
+
 [[append-processor]]
 === Append Processor
 Appends one or more values to an existing array if the field already exists and it is an array.
@@ -778,6 +784,7 @@ Accepts a single value or an array of values.
 | Name      | Required  | Default  | Description
 | `field`  | yes       | -        | The field to be appended to. Supports <<accessing-template-fields,template snippets>>.
 | `value`  | yes       | -        | The value to be appended. Supports <<accessing-template-fields,template snippets>>.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -806,6 +813,7 @@ the field is not a supported format or resultant value exceeds 2^63.
 | `field`          | yes       | -        | The field to convert
 | `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -844,6 +852,7 @@ still be updated with the unconverted field value.
 | `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `type`           | yes       | -        | The type to convert the existing value to
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -876,6 +885,7 @@ in the same order they were defined as part of the processor definition.
 | `formats`              | yes       | -                   | An array of the expected date formats. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
 | `timezone`        | no        | UTC                 | The timezone to use when parsing the date. Supports <<accessing-template-fields,template snippets>>.
 | `locale`          | no        | ENGLISH             | The locale to use when parsing the date, relevant when parsing month names or week days. Supports <<accessing-template-fields,template snippets>>.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 Here is an example that adds the parsed date to the `timestamp` field based on the `initial_date` field:
@@ -1065,6 +1075,7 @@ understands this to mean `2016-04-01` as is explained in the <<date-math-index-n
 | `timezone`             | no        | UTC                          | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names.
 | `locale`               | no        | ENGLISH                      | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days.
 | `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid Joda pattern is expected here. Supports <<accessing-template-fields,template snippets>>.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [[dissect-processor]]
@@ -1134,7 +1145,7 @@ See <<dissect-key-modifiers, below>> for more information.
 | `pattern`         | yes       | -        | The pattern to apply to the field
 | `append_separator`| no        | "" (empty string) | The character(s) that separate the appended fields.
 | `ignore_missing`  | no        | false             | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
-| `
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -1162,7 +1173,6 @@ modifiers.
 | `+` with `/n` | Append with order  | left and right | `%{+keyname/2} %{+keyname/1}` | Appends two or more fields together in the order specified   | <<dissect-modifier-append-key-with-order,link>>
 | `?`           | Named skip key     | left           | `%{?ignoreme}`  | Skips the matched value in the output. Same behavior as `%{}`| <<dissect-modifier-named-skip-key,link>>
 | `*` and `&`   | Reference keys     | left           | `%{*r1} %{&r1}`    | Sets the output key as value of `*` and output value of `&`  | <<dissect-modifier-reference-keys,link>>
-| `
 |======
 
 [[dissect-modifier-skip-right-padding]]
@@ -1265,6 +1275,14 @@ Reference key modifier example
 Drops the document without raising any errors. This is useful to prevent the document from
 getting indexed based on some condition.
 
+[[drop-options]]
+.Drop Options
+[options="header"]
+|======
+| Name              | Required  | Default  | Description
+include::ingest-node-common-processor.asciidoc[]
+|======
+
 [source,js]
 --------------------------------------------------
 {
@@ -1289,6 +1307,7 @@ Otherwise these <<accessing-data-in-pipelines,fields>> can't be accessed by any
 | Name     | Required  | Default  | Description
 | `field`  | yes       | -        | The field to expand into an object field
 | `path`   | no        | -        | The field that contains the field to expand. Only required if the field to expand is part another object field, because the `field` option can only understand leaf fields.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -1406,6 +1425,7 @@ to the requester.
 |======
 | Name       | Required  | Default  | Description
 | `message`  | yes       | -        | The error message thrown by the processor. Supports <<accessing-template-fields,template snippets>>.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -1452,6 +1472,7 @@ then it aborts the execution and leaves the array unmodified.
 | `field`          | yes       | -        | The array field
 | `processor`      | yes       | -        | The processor to execute against each field
 | `ignore_missing` | no        | false    | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 Assume the following document:
@@ -1645,6 +1666,7 @@ Grok expression.
 | `pattern_definitions`  | no        | -                   | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
 | `trace_match`          | no        | false               | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
 | `ignore_missing`       | no        | false               | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@@ -1919,6 +1941,7 @@ If the field is not a string, the processor will throw an exception.
 | `replacement`     | yes       | -        | The string to replace the matching patterns with
 | `target_field`    | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -1946,6 +1969,7 @@ Throws an error when the field is not an array.
 | `field`           | yes       | -        | The field to be separated
 | `separator`       | yes       | -        | The separator character
 | `target_field`    | no        | `field`  | The field to assign the joined value to, by default `field` is updated in-place
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -1971,6 +1995,7 @@ Converts a JSON string into a structured JSON object.
 | `field`        | yes       | -        | The field to be parsed
 | `target_field` | no        | `field`  | The field to insert the converted structured object into
 | `add_to_root`  | no        | false    | Flag that forces the serialized json to be injected into the top level of the document. `target_field` must not be set when this option is chosen.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 All JSON-supported types will be parsed (null, boolean, number, array, object, string).
@@ -2082,6 +2107,7 @@ For example, if you have a log message which contains `ip=1.2.3.4 error=REFUSED`
 | `trim_key`       | no        | `null`   | String of characters to trim from extracted keys
 | `trim_value`     | no        | `null`   | String of characters to trim from extracted values
 | `strip_brackets` | no        | `false`  | If `true` strip brackets `()`, `<>`, `[]` as well as quotes `'` and `"` from extracted values
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 
@@ -2097,6 +2123,7 @@ Converts a string to its lowercase equivalent.
 | `field`          | yes       | -        | The field to make lowercase
 | `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2119,6 +2146,7 @@ Executes another pipeline.
 |======
 | Name             | Required  | Default  | Description
 | `name`           | yes       | -        | The name of the pipeline to execute
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2234,6 +2262,7 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown
 | Name             | Required  | Default  | Description
 | `field`          | yes       | -        | Fields to be removed. Supports <<accessing-template-fields,template snippets>>.
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 Here is an example to remove a single field:
@@ -2272,6 +2301,7 @@ Renames an existing field. If the field doesn't exist or the new name is already
 | `field`           | yes       | -        | The field to be renamed. Supports <<accessing-template-fields,template snippets>>.
 | `target_field`    | yes       | -        | The new name of the field. Supports <<accessing-template-fields,template snippets>>.
 | `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2305,6 +2335,7 @@ caching see <<modules-scripting-using-caching, Script Caching>>.
 | `id`                   | no        | -          | The stored script id to refer to
 | `source`               | no        | -          | An inline script to be executed
 | `params`               | no        | -          | Script Parameters
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 One of `id` or `source` options must be provided in order to properly reference a script to execute.
@@ -2401,6 +2432,7 @@ its value will be replaced with the provided one.
 | `field` | yes       | -        | The field to insert, upsert, or update. Supports <<accessing-template-fields,template snippets>>.
 | `value` | yes       | -        | The value to be set for the field. Supports <<accessing-template-fields,template snippets>>.
 | `override` | no        | true     | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched.
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2414,6 +2446,43 @@ its value will be replaced with the provided one.
 --------------------------------------------------
 // NOTCONSOLE
 
+
+[[ingest-node-set-security-user-processor]]
+=== Set Security User Processor
+Sets user-related details (such as `username`,  `roles`, `email`, `full_name`
+and `metadata` ) from the current
+authenticated user to the current document by pre-processing the ingest.
+
+IMPORTANT: Requires an authenticated user for the index request.
+
+[[set-security-user-options]]
+.Set Security User Options
+[options="header"]
+|======
+| Name          | Required  | Default                                                   | Description
+| `field`       | yes       | -                                                         | The field to store the user information into.
+| `properties`  | no        | [`username`, `roles`, `email`, `full_name`, `metadata`]   | Controls what user related properties are added to the `field`.
+include::ingest-node-common-processor.asciidoc[]
+|======
+
+The following example adds all user details for the current authenticated user
+to the `user` field for all documents that are processed by this pipeline:
+
+[source,js]
+--------------------------------------------------
+{
+  "processors" : [
+    {
+      "set_security_user": {
+        "field": "user"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+
 [[split-processor]]
 === Split Processor
 Splits a field into an array using a separator character. Only works on string fields.
@@ -2427,6 +2496,7 @@ Splits a field into an array using a separator character. Only works on string f
 | `separator`       | yes       | -        | A regex which matches the separator, eg `,` or `\s+`
 | `target_field`    | no        | `field`  | The field to assign the split value to, by default `field` is updated in-place
 | `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2455,6 +2525,7 @@ Throws an error when the field is not an array.
 | `field`           | yes       | -        | The field to be sorted
 | `order`           | no        | `"asc"`  | The sort order to use. Accepts `"asc"` or `"desc"`.
 | `target_field`    | no        | `field`  | The field to assign the sorted value to, by default `field` is updated in-place
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2482,6 +2553,7 @@ NOTE: This only works on leading and trailing whitespace.
 | `field`           | yes       | -        | The string-valued field to trim whitespace from
 | `target_field`    | no        | `field`  | The field to assign the trimmed value to, by default `field` is updated in-place
 | `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2506,6 +2578,7 @@ Converts a string to its uppercase equivalent.
 | `field`          | yes       | -        | The field to make uppercase
 | `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
@@ -2530,6 +2603,7 @@ URL-decodes a string
 | `field`          | yes       | -        | The field to decode
 | `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+include::ingest-node-common-processor.asciidoc[]
 |======
 
 [source,js]
diff --git a/docs/reference/migration/migrate_6_5.asciidoc b/docs/reference/migration/migrate_6_5.asciidoc
index 4f3716b2ae4a7..a4dab4dc109a3 100644
--- a/docs/reference/migration/migrate_6_5.asciidoc
+++ b/docs/reference/migration/migrate_6_5.asciidoc
@@ -95,11 +95,13 @@ Entries in the previous format contained positional fields.
 These fields are no longer positional in the new format, as they are part of the
 structured log entry (the JSON document).
 Consequently, the following settings, which toggled some of the prefix
-positional fields, have be renamed ("prefix" was spliced out):
+positional fields, have been renamed ("prefix" was spliced out):
+
 - `xpack.security.audit.logfile.prefix.emit_node_host_address` to
   `xpack.security.audit.logfile.emit_node_host_address`
 - `xpack.security.audit.logfile.prefix.emit_node_host_name` to
   `xpack.security.audit.logfile.emit_node_host_name`
 - `xpack.security.audit.logfile.prefix.emit_node_name` to
   `xpack.security.audit.logfile.emit_node_name`
+
 In 7.0.0 the `prefix` variant will be removed.
diff --git a/docs/reference/modules/indices/query_cache.asciidoc b/docs/reference/modules/indices/query_cache.asciidoc
index f6cdf71925a94..aaa1ab1742841 100644
--- a/docs/reference/modules/indices/query_cache.asciidoc
+++ b/docs/reference/modules/indices/query_cache.asciidoc
@@ -5,6 +5,7 @@ The query cache is responsible for caching the results of queries.
 There is one queries cache per node that is shared by all shards.
 The cache implements an LRU eviction policy: when a cache becomes full, the
 least recently used data is evicted to make way for new data.
+It is not possible to look at the contents being cached.
 
 The query cache only caches queries which are being used in a filter context.
 
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index 81d882f5f0eb6..4bf3073abd359 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -152,6 +152,15 @@ PUT _cluster/settings
   by default, but they can selectively be made optional by setting this setting
   to `true`.
 
+`cluster.remote.${cluster_alias}.transport.ping_schedule`::
+
+  Sets the time interval between regular application-level ping messages that
+  are sent to ensure that transport connections to nodes belonging to remote
+  clusters are kept alive. If set to `-1`, application-level ping messages to
+  this remote cluster are not sent. If unset, application-level ping messages
+  are sent according to the global `transport.ping_schedule` setting, which
+  defaults to ``-1` meaning that pings are not sent.
+
 [float]
 [[retrieve-remote-clusters-info]]
 === Retrieving remote clusters info
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 725808e193640..86202a98dd537 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -204,7 +204,7 @@ you can change this behavior by using the `script.cache.expire` setting.
 You can configure the size of this cache by using the `script.cache.max_size` setting.
 By default, the cache size is `100`.
 
-NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
+NOTE: The size of scripts is limited to 65,535 bytes. This can be
 changed by setting `script.max_size_in_bytes` setting to increase that soft
 limit, but if scripts are really large then a
 <<modules-scripting-engine,native script engine>> should be considered.
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index b2edca480117b..0aab3665951aa 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -112,8 +112,8 @@ which returns:
 -----------------------------------
 // TESTRESPONSE
 
-To retrieve information about multiple repositories, specify a
-a comma-delimited list of repositories. You can also use the * wildcard when
+To retrieve information about multiple repositories, specify a comma-delimited 
+list of repositories. You can also use the * wildcard when
 specifying repository names. For example, the following request retrieves
 information about all of the snapshot repositories that start with `repo` or
 contain `backup`:
diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc
index 257181f70c507..c1bc83230e597 100644
--- a/docs/reference/modules/transport.asciidoc
+++ b/docs/reference/modules/transport.asciidoc
@@ -46,9 +46,9 @@ between all nodes. Defaults to `false`.
 
 |`transport.ping_schedule` | Schedule a regular application-level ping message
 to ensure that transport connections between nodes are kept alive. Defaults to
-`5s` in the transport client and `-1` (disabled) elsewhere. It is preferable to
-correctly configure TCP keep-alives instead of using this feature, because TCP
-keep-alives apply to all kinds of long-lived connection and not just to
+`5s` in the transport client and `-1` (disabled) elsewhere. It is preferable
+to correctly configure TCP keep-alives instead of using this feature, because
+TCP keep-alives apply to all kinds of long-lived connections and not just to
 transport connections.
 
 |=======================================================================
diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc
index 3d4a37861f1ae..34cbee9c1699f 100644
--- a/docs/reference/monitoring/configuring-metricbeat.asciidoc
+++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc
@@ -3,7 +3,9 @@
 [[configuring-metricbeat]]
 === Monitoring {es} with {metricbeat}
 
-beta[] In 6.5 and later, you can use {metricbeat} to collect data about {es} 
+beta[]
+
+In 6.5 and later, you can use {metricbeat} to collect data about {es} 
 and ship it to the monitoring cluster, rather than routing it through exporters 
 as described in <<configuring-monitoring>>. 
 
diff --git a/docs/reference/monitoring/images/metricbeat.png b/docs/reference/monitoring/images/metricbeat.png
index bf6434dc4b40c..f74f856653043 100644
Binary files a/docs/reference/monitoring/images/metricbeat.png and b/docs/reference/monitoring/images/metricbeat.png differ
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 4b4a82594a110..7c5ca95623e83 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -208,10 +208,10 @@ not. The number value is of type float.
 [[function-random]]
 ==== Random
 
-The `random_score` generates scores that are uniformly distributed in [0, 1[.
-By default, it uses the internal Lucene doc ids as a source of randomness,
-which is very efficient but unfortunately not reproducible since documents might
-be renumbered by merges.
+The `random_score` generates scores that are uniformly distributed from 0 up to
+but not including 1. By default, it uses the internal Lucene doc ids as a
+source of randomness, which is very efficient but unfortunately not
+reproducible since documents might be renumbered by merges.
 
 In case you want scores to be reproducible, it is possible to provide a `seed`
 and `field`. The final score will then be computed based on this seed, the
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 189f135fa3bc7..c33b227824bdf 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -1,7 +1,7 @@
 [[query-dsl-geo-polygon-query]]
 === Geo Polygon Query
 
-A query allowing to include hits that only fall within a polygon of
+A query returning hits that only fall within a polygon of
 points. Here is an example:
 
 [source,js]
@@ -17,9 +17,9 @@ GET /_search
                 "geo_polygon" : {
                     "person.location" : {
                         "points" : [
-                        {"lat" : 40, "lon" : -70},
-                        {"lat" : 30, "lon" : -80},
-                        {"lat" : 20, "lon" : -90}
+                            {"lat" : 40, "lon" : -70},
+                            {"lat" : 30, "lon" : -80},
+                            {"lat" : 20, "lon" : -90}
                         ]
                     }
                 }
@@ -49,7 +49,9 @@ or longitude, or `STRICT` (default is `STRICT`).
 [float]
 ===== Lat Long as Array
 
-Format in `[lon, lat]`, note, the order of lon/lat here in order to
+Format as `[lon, lat]`
+
+Note: the order of lon/lat here must
 conform with http://geojson.org/[GeoJSON].
 
 [source,js]
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index aac57363ffa60..2a9a9de4851d4 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -118,6 +118,10 @@ The above request will yield the following response:
 <2> The `_percolator_document_slot` field indicates which document has matched with this query.
     Useful when percolating multiple document simultaneously.
 
+TIP: To provide a simple example, this documentation uses one index `my-index` for both the percolate queries and documents.
+This set-up can work well when there are just a few percolate queries registered. However, with heavier usage it is recommended
+to store queries and documents in separate indices. Please see <<how-it-works, How it Works Under the Hood>> for more details.
+
 [float]
 ==== Parameters
 
@@ -643,6 +647,7 @@ The above search request returns a response similar to this:
     query with `_name` parameter set to `query1`.
 
 [float]
+[[how-it-works]]
 ==== How it Works Under the Hood
 
 When indexing a document into an index that has the <<percolator,percolator field type>> mapping configured, the query
@@ -679,3 +684,11 @@ GET /_search
 
 NOTE: The above example assumes that there is a `query` field of type
 `percolator` in the mappings.
+
+Given the design of percolation, it often makes sense to use separate indices for the percolate queries and documents
+being percolated, as opposed to a single index as we do in examples. There are a few benefits to this approach:
+
+- Because percolate queries contain a different set of fields from the percolated documents, using two separate indices
+allows for fields to be stored in a denser, more efficient way.
+- Percolate queries do not scale in the same way as other queries, so percolation performance may benefit from using
+a different index configuration, like the number of primary shards.
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index 620a175ff39a5..c5087d52f905e 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -139,3 +139,26 @@ GET _search
 // CONSOLE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
 <2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
+
+[[querying-range-fields]]
+==== Querying range fields
+
+`range` queries can be used on fields of type <<range,`range`>>, allowing to
+match a range specified in the query with a range field value in the document.
+The `relation` parameter controls how these two ranges are matched:
+
+[horizontal]
+`WITHIN`::
+
+    Matches documents who's range field is entirely within the query's range.
+
+`CONTAINS`::
+
+    Matches documents who's range field entirely contains the query's range.
+
+`INTERSECTS`::
+
+    Matches documents who's range field intersects the query's range.
+    This is the default value when querying range fields.
+
+For examples, see <<range,`range`>> mapping type.
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 4a40e4dcf440e..9b7c776f8b039 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -6,6 +6,7 @@
 This section summarizes the changes in each release.
 
 * <<release-notes-6.5.0>>
+* <<release-notes-6.4.3>>
 * <<release-notes-6.4.2>>
 * <<release-notes-6.4.1>>
 * <<release-notes-6.4.0>>
diff --git a/docs/reference/release-notes/6.4.asciidoc b/docs/reference/release-notes/6.4.asciidoc
index cefb85042347c..0631e9b9567d0 100644
--- a/docs/reference/release-notes/6.4.asciidoc
+++ b/docs/reference/release-notes/6.4.asciidoc
@@ -35,18 +35,28 @@
 ////
 
 [[release-notes-6.4.3]]
-== 6.4.3 Release Notes
+== {es} version 6.4.3
 
 coming[6.4.3]
 
 Also see <<breaking-changes-6.4>>.
 
+[float]
+[[enhancement-6.4.3]]
+=== Enhancements
+
+Machine learning::
+* Changes linker options on macOS to allow Homebrew installs ({ml-pull}225[#225])
+
 [[bug-6.4.3]]
 [float]
 === Bug fixes
 
+Aggregations::
+* Check self references in metric agg after last doc collection (#33593) {pull}34001[#34001]
+
 Authentication::
-* ListenableFuture should preserve ThreadContext {pull}34394[#34394]
+* ListenableFuture should preserve ThreadContext (CVE-2018-17244) {pull}34394[#34394]
 * Allow an AuthenticationResult to return metadata {pull}34382[#34382] (issues: {issue}34290[#34290], {issue}34332[#34332])
 * Preserve thread context during authentication  {pull}34290[#34290]
 
@@ -56,6 +66,13 @@ Circuit Breakers::
 Java High Level REST Client::
 * HLRC: Fixing bug when getting a missing pipeline {pull}34286[#34286] (issue: {issue}34119[#34119])
 
+Machine learning::
+* Fixes the cause of `hard_limit` memory errors for jobs with bucket spans greater 
+than one day ({ml-pull}243[#243])
+* Rules that trigger the `skip_model_update` action should also apply to the 
+anomaly model. This fixes an issue where anomaly scores of results that triggered 
+the rule would decrease if they occurred frequently. {ml-pull}222[#222] (issue:{ml-issue}217[#217])
+
 Network::
 *  Support PKCS#11 tokens as keystores and truststores  {pull}34063[#34063] (issue: {issue}11[#11])
 * Correctly handle PKCS#11 tokens for system keystore {pull}33460[#33460] (issues: {issue}11[#11], {issue}33459[#33459])
diff --git a/docs/reference/release-notes/6.5.asciidoc b/docs/reference/release-notes/6.5.asciidoc
index 06473d1991f49..b3adc097f6b72 100644
--- a/docs/reference/release-notes/6.5.asciidoc
+++ b/docs/reference/release-notes/6.5.asciidoc
@@ -95,9 +95,6 @@ Plugins::
 Analysis::
 * Deprecates the standard filter {pull}33468[#33468]
 
-Geo::
-* Make Geo Context Mapping Parsing More Strict (6.x version) {pull}32862[#32862] (issue: {issue}32821[#32821])
-
 Scripting::
 * Scripting: Conditionally use java time api in scripting {pull}31441[#31441]
 
@@ -105,6 +102,9 @@ Search::
 * Deprecate negative query boost (#34486) {pull}34512[#34512] (issues: {issue}33309[#33309], {issue}34486[#34486])
 * Deprecate negative `weight` in Function Score Query {pull}33624[#33624] (issue: {issue}31927[#31927])
 
+Suggesters::
+* Make Geo Context Mapping Parsing More Strict (6.x version) {pull}32862[#32862] (issue: {issue}32821[#32821])
+
 
 
 [[feature-6.5.0]]
@@ -124,11 +124,11 @@ Audit::
 
 Authentication::
 * Allow User/Password realms to disable authc {pull}34033[#34033] (issue: {issue}33292[#33292])
-* Add support for "authorization_realms" {pull}33262[#33262]
 
 Authorization::
 * [X-Pack] Beats centralized management: security role + licensing {pull}34305[#34305] (issues: {issue}30493[#30493], {issue}30520[#30520])
 * Add get-user-privileges API {pull}33928[#33928] (issue: {issue}32777[#32777])
+* Add support for "authorization_realms" {pull}33262[#33262]
 
 CCR::
 * [CCR] Added auto follow patterns feature {pull}33118[#33118] (issue: {issue}33007[#33007])
@@ -194,13 +194,9 @@ Authorization::
 * [Kerberos] Add authorization realms support to Kerberos realm {pull}32392[#32392]
 * [X-Pack] Beats centralized management: security role + licensing {pull}30520[#30520] (issue: {issue}30493[#30493])
 
-Beats::
+Beats Plugin::
 * [Monitoring] Update beats template to include apm-server metrics {pull}33286[#33286]
 
-CCR::
-* Fill LocalCheckpointTracker with Lucene commit {pull}34474[#34474] (issues: {issue}0[#0], {issue}2[#2], {issue}33656[#33656])
-* Integrates soft-deletes into Elasticsearch {pull}33222[#33222] (issues: {issue}29530[#29530], {issue}30086[#30086], {issue}30120[#30120], {issue}30335[#30335], {issue}30522[#30522], {issue}31106[#31106])
-
 CRUD::
 * Verify primary mode usage with assertions {pull}32667[#32667] (issues: {issue}10708[#10708], {issue}25692[#25692], {issue}32442[#32442])
 * Refactor TransportShardBulkAction to better support retries {pull}31821[#31821]
@@ -222,10 +218,11 @@ Distributed::
 * Expose whether or not the global checkpoint updated {pull}32659[#32659] (issue: {issue}32651[#32651])
 * Include translog path in error message when translog is corrupted {pull}32251[#32251] (issue: {issue}24929[#24929])
 
-Docs::
+Docs Infrastructure::
 * Docs: Allow snippets to have line continuation {pull}32649[#32649]
 
 Engine::
+* Fill LocalCheckpointTracker with Lucene commit {pull}34474[#34474] (issues: {issue}0[#0], {issue}2[#2], {issue}33656[#33656])
 * Lock down Engine.Searcher {pull}34363[#34363] (issue: {issue}34357[#34357])
 * Fold EngineSearcher into Engine.Searcher {pull}34082[#34082]
 * Build DocStats from SegmentInfos in ReadOnlyEngine {pull}34079[#34079] (issue: {issue}33903[#33903])
@@ -235,7 +232,6 @@ Engine::
 * Allow engine to recover from translog upto a seqno {pull}33032[#33032] (issue: {issue}32867[#32867])
 
 ILM::
-* HLRC: Add ILM Retry {pull}33990[#33990] (issue: {issue}33100[#33100])
 * 6.x - HLRC: Add ILM Status to HLRC (#33283) {pull}33448[#33448] (issue: {issue}33283[#33283])
 
 Index APIs::
@@ -378,12 +374,17 @@ Recovery::
 * Restore local history from translog on promotion {pull}33616[#33616] (issues: {issue}32867[#32867], {issue}33473[#33473])
 * Reset replica engine to global checkpoint on promotion {pull}33473[#33473] (issue: {issue}32867[#32867])
 * Bootstrap a new history_uuid when force allocating a stale primary {pull}33432[#33432] (issue: {issue}26712[#26712])
+* Integrates soft-deletes into Elasticsearch {pull}33222[#33222] (issues: {issue}29530[#29530], {issue}30086[#30086], {issue}30120[#30120], {issue}30335[#30335], {issue}30522[#30522], {issue}31106[#31106])
 
 Rollup::
 * [Rollup] Add support for date histo `format` when searching {pull}34537[#34537] (issue: {issue}34391[#34391])
 * [Rollup] Only allow aggregating on multiples of configured interval {pull}32052[#32052]
 
 SQL::
+* SQL: Optimizer rule for folding nullable expressions {pull}35080[#35080] (issue: {issue}34826[#34826])
+* SQL: Improve painless script generated from `IN` {pull}35055[#35055] (issue: {issue}34750[#34750])
+* SQL: Implement CAST between STRING and IP {pull}34949[#34949] (issue: {issue}34799[#34799])
+* SQL: Fix function args verification and error msgs {pull}34926[#34926] (issues: {issue}33469[#33469], {issue}34752[#34752])
 * SQL: Introduce ODBC mode, similar to JDBC {pull}34825[#34825] (issue: {issue}34720[#34720])
 * SQL: Introduce support for IP fields {pull}34758[#34758] (issue: {issue}32499[#32499])
 * SQL: Implement null handling for `IN(v1, v2, ...)` {pull}34750[#34750] (issue: {issue}34582[#34582])
@@ -430,6 +431,7 @@ Search::
 * Ignore script fields when size is 0 {pull}31917[#31917] (issue: {issue}31824[#31824])
 
 Security::
+* Generate non-encrypted license public key {pull}34626[#34626]
 * Security: don't call prepare index for reads {pull}34568[#34568] (issues: {issue}33205[#33205], {issue}34246[#34246])
 * Enable security automaton caching {pull}34028[#34028]
 * Add Debug/Trace logging to token service {pull}34022[#34022]
@@ -474,6 +476,7 @@ ZenDiscovery::
 
 Aggregations::
 * Fix handling of empty keyword in terms aggregation {pull}34457[#34457] (issue: {issue}34434[#34434])
+* Check self references in metric agg after last doc collection (#33593) {pull}34001[#34001]
 * Unmapped aggs should not run pipelines if they delegate reduction {pull}33528[#33528] (issue: {issue}33514[#33514])
 * For filters aggregations, make sure that rewrites preserve other_bucket. {pull}32921[#32921] (issue: {issue}32834[#32834])
 
@@ -529,6 +532,7 @@ Geo::
 * Use the determinant formula for calculating the orientation of a polygon {pull}27967[#27967]
 
 Index APIs::
+* Make XContentBuilder in AliasActions build `is_write_index` field {pull}35071[#35071]
 * Do not update number of replicas on no indices {pull}34481[#34481]
 * [Security] Get Alias API wildcard exclusion with Security {pull}34144[#34144] (issues: {issue}33518[#33518], {issue}33805[#33805])
 * Allow to clear the fielddata cache per field {pull}33807[#33807] (issue: {issue}33798[#33798])
@@ -558,9 +562,9 @@ Machine Learning::
 * Handle pre-6.x time fields {pull}34373[#34373]
 * [ML] Get job stats request should filter non-ML job tasks {pull}33516[#33516] (issue: {issue}33515[#33515])
 * [ML] Prevent NPE parsing the stop datafeed request. {pull}33347[#33347]
+* [ML] fix updating opened jobs scheduled events (#31651) {pull}32881[#32881] (issue: {issue}31651[#31651])
 * Clear Job#finished_time when it is opened (#32605) {pull}32755[#32755]
 * [ML] Fix thread leak when waiting for job flush (#32196) {pull}32541[#32541] (issue: {issue}32196[#32196])
-* [ML] Adding an open job to a group does not update autodetect with related scheduled events {pull}31651[#31651]
 
 Mapping::
 * Fix field mapping updates with similarity {pull}33634[#33634] (issue: {issue}33611[#33611])
@@ -588,11 +592,15 @@ Packaging::
 REST API::
 * Core: Fix IndicesSegmentResponse.toXcontent() serialization {pull}33414[#33414] (issue: {issue}29120[#29120])
 
+Ranking::
+* Fix a bug in function_score queries where we use the wrong boost_mode. {pull}35148[#35148] (issue: {issue}35123[#35123])
+
 Recovery::
 * Resync fails to notify on unavaiable exceptions {pull}33615[#33615] (issues: {issue}31179[#31179], {issue}33613[#33613])
 * Ensure to generate identical NoOp for the same failure {pull}33141[#33141] (issue: {issue}32986[#32986])
 
 Rollup::
+* [Rollup] Proactively resolve index patterns in RollupSearch endoint {pull}34930[#34930] (issue: {issue}34828[#34828])
 * Address BWC bug due to default metrics in (#34764) {pull}34810[#34810] (issue: {issue}34764[#34764])
 * Allowing {index}/_xpack/rollup/data to accept comma delimited list {pull}34115[#34115]
 * [Rollup] Fix Caps Comparator to handle calendar/fixed time {pull}33336[#33336] (issue: {issue}32052[#32052])
@@ -601,6 +609,15 @@ Rollup::
 * [Rollup] Improve ID scheme for rollup documents {pull}32558[#32558] (issue: {issue}32372[#32372])
 
 SQL::
+* SQL: handle wildcard expansion on incorrect fields {pull}35134[#35134] (issue: {issue}35092[#35092])
+* SQL: Register missing processors {pull}35121[#35121] (issue: {issue}35119[#35119])
+* SQL: Fix NPE thrown if HAVING filter evals to null {pull}35108[#35108] (issue: {issue}35107[#35107])
+* SQL: Proper handling of nested fields at the beginning of the columns list {pull}35068[#35068] (issue: {issue}32951[#32951])
+* SQL: Fix incorrect AVG data type {pull}34948[#34948] (issue: {issue}33773[#33773])
+* SQL: Add `CAST` and `CONVERT` to `SHOW FUNCTIONS` {pull}34940[#34940] (issue: {issue}34939[#34939])
+* SQL: Handle aggregation for null group {pull}34916[#34916] (issue: {issue}34896[#34896])
+* SQL: Provide null-safe scripts for Not and Neg {pull}34877[#34877] (issue: {issue}34848[#34848])
+* SQL: Return error with ORDER BY on non-grouped. {pull}34855[#34855] (issue: {issue}34590[#34590])
 * SQL: Fix queries with filter resulting in NO_MATCH {pull}34812[#34812] (issue: {issue}34613[#34613])
 * SQL: Fix edge case: `<field> IN (null)` {pull}34802[#34802] (issue: {issue}34750[#34750])
 * SQL: Verifier allows aliases aggregates for sorting {pull}34773[#34773] (issue: {issue}34607[#34607])
diff --git a/docs/reference/release-notes/highlights-6.5.0.asciidoc b/docs/reference/release-notes/highlights-6.5.0.asciidoc
index affe5946c346a..e23a1822aa54f 100644
--- a/docs/reference/release-notes/highlights-6.5.0.asciidoc
+++ b/docs/reference/release-notes/highlights-6.5.0.asciidoc
@@ -4,4 +4,6 @@
 <titleabbrev>6.5.0</titleabbrev>
 ++++
 
-coming[6.5.0]
\ No newline at end of file
+coming[6.5.0]
+
+See also <<release-notes-6.5.0,{es} 6.5.0 release notes>>. 
\ No newline at end of file
diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
index 4fe169a1655e0..85fda6152919e 100644
--- a/docs/reference/release-notes/highlights.asciidoc
+++ b/docs/reference/release-notes/highlights.asciidoc
@@ -9,10 +9,12 @@
 This section summarizes the most important changes in each release. For the 
 full list, see <<es-release-notes>> and <<breaking-changes>>. 
 
+* <<release-highlights-6.5.0>>
 * <<release-highlights-6.4.0>>
 * <<release-highlights-6.3.0>>
 
 --
 
+include::highlights-6.5.0.asciidoc[]
 include::highlights-6.4.0.asciidoc[]
 include::highlights-6.3.0.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc
index 5aef27e127500..eedc2dfa1f51f 100644
--- a/docs/reference/rest-api/index.asciidoc
+++ b/docs/reference/rest-api/index.asciidoc
@@ -10,6 +10,7 @@ directly to configure and access {xpack} features.
 * <<info-api,Info API>>
 * <<ccr-apis,Cross-cluster replication APIs>>
 * <<graph-explore-api,Graph Explore API>>
+* <<index-lifecycle-management-api,Index lifecycle management APIs>>
 * <<licensing-apis,Licensing APIs>>
 * <<ml-apis,Machine Learning APIs>>
 * <<security-api,Security APIs>>
@@ -22,6 +23,7 @@ directly to configure and access {xpack} features.
 include::info.asciidoc[]
 include::{es-repo-dir}/ccr/apis/ccr-apis.asciidoc[]
 include::{es-repo-dir}/graph/explore.asciidoc[]
+include::{es-repo-dir}/ilm/apis/ilm-api.asciidoc[]
 include::{es-repo-dir}/licensing/index.asciidoc[]
 include::{es-repo-dir}/migration/migration.asciidoc[]
 include::{es-repo-dir}/ml/apis/ml-api.asciidoc[]
diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc
index 6d5967d0cdc7d..fc6488010dabd 100644
--- a/docs/reference/rest-api/info.asciidoc
+++ b/docs/reference/rest-api/info.asciidoc
@@ -68,6 +68,11 @@ Example response:
          "available" : true,
          "enabled" : true
       },
+      "ilm" : {
+               "description" : "Index lifecycle management for the Elastic Stack",
+               "available" : true,
+               "enabled" : true
+      },
       "logstash" : {
          "description" : "Logstash management component for X-Pack",
          "available" : true,
diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc
index 8e7fc69a00a6b..e2252a772184d 100644
--- a/docs/reference/rollup/apis/rollup-search.asciidoc
+++ b/docs/reference/rollup/apis/rollup-search.asciidoc
@@ -30,6 +30,7 @@ Rules for the `index` parameter:
 or using `_all`, is not permitted
 - Multiple non-rollup indices may be specified
 - Only one rollup index may be specified.  If more than one are supplied an exception will be thrown
+- Index patterns may be used, but if they match more than one rollup index an exception will be thrown.
 
 ==== Request Body
 
diff --git a/docs/reference/rollup/rollup-search-limitations.asciidoc b/docs/reference/rollup/rollup-search-limitations.asciidoc
index b61d1a743880f..c8a736450bde0 100644
--- a/docs/reference/rollup/rollup-search-limitations.asciidoc
+++ b/docs/reference/rollup/rollup-search-limitations.asciidoc
@@ -21,6 +21,7 @@ follows:
 or using `_all`, is not permitted
 - Multiple non-rollup indices may be specified
 - Only one rollup index may be specified.  If more than one are supplied an exception will be thrown
+- Index patterns may be used, but if they match more than one rollup index an exception will be thrown.
 
 This limitation is driven by the logic that decides which jobs are the "best" for any given query.  If you have ten jobs stored in a single
 index, which cover the source data with varying degrees of completeness and different intervals, the query needs to determine which set
diff --git a/docs/reference/settings/audit-settings.asciidoc b/docs/reference/settings/audit-settings.asciidoc
index ec661a1f30c9f..69045dca0a2db 100644
--- a/docs/reference/settings/audit-settings.asciidoc
+++ b/docs/reference/settings/audit-settings.asciidoc
@@ -22,14 +22,13 @@ file named `<clustername>_audit.log` on each node.
 You can also specify `index`, which puts the auditing events in an {es} index
 that is prefixed with `.security_audit_log`. The index can reside on the same
 cluster or a separate cluster.
-
++
 For backwards compatibility reasons, if you use the logfile output type, a
 `<clustername>_access.log` file is also created. It contains the same
 information, but it uses the older (pre-6.5.0) formatting style.
 If the backwards compatible format is not required, it should be disabled.
 To do that, change its logger level to `off` in the `log4j2.properties` file.
 For more information, see <<configuring-logging-levels>>.
-
 +
 --
 TIP: If the index is unavailable, it is possible for auditing events to
diff --git a/docs/reference/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc
index f75bf3a9f2986..5d4f9519006ae 100644
--- a/docs/reference/settings/ml-settings.asciidoc
+++ b/docs/reference/settings/ml-settings.asciidoc
@@ -77,11 +77,11 @@ opening spend more time in the `opening` state. Defaults to `2`.
 These settings are for advanced use cases; the default values are generally 
 sufficient:
 
-`xpack.ml.max_anomaly_records`:: (<<cluster-update-settings,Dynamic>>) 
+`xpack.ml.max_anomaly_records` (<<cluster-update-settings,Dynamic>>)::
 The maximum number of records that are output per bucket. The default value is 
 `500`.
 
-`xpack.ml.max_lazy_ml_nodes`:: (<<cluster-update-settings,Dynamic>>)
+`xpack.ml.max_lazy_ml_nodes` (<<cluster-update-settings,Dynamic>>)::
 The number of lazily spun up Machine Learning nodes. Useful in situations
 where ML nodes are not desired until the first Machine Learning Job
 is opened. It defaults to `0` and has a maximum acceptable value of `3`.
diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc
index aac6d15cd1c4e..f2a12c1e4b6c4 100644
--- a/docs/reference/settings/monitoring-settings.asciidoc
+++ b/docs/reference/settings/monitoring-settings.asciidoc
@@ -43,14 +43,14 @@ to `true`. Its default value is `false`.
 The `xpack.monitoring.collection` settings control how data is collected from
 your Elasticsearch nodes.
 
-`xpack.monitoring.collection.enabled`:: (<<cluster-update-settings,Dynamic>>)
+`xpack.monitoring.collection.enabled` (<<cluster-update-settings,Dynamic>>)::
 
 added[6.3.0] Set to `true` to enable the collection of monitoring data. When
 this setting is `false` (default), {es} monitoring data is not collected and
 all monitoring data from other sources such as {kib}, Beats, and Logstash is
 ignored.
 
-`xpack.monitoring.collection.interval`:: (<<cluster-update-settings,Dynamic>>) 
+`xpack.monitoring.collection.interval` (<<cluster-update-settings,Dynamic>>):: 
 
 Setting to `-1` to disable data collection has been deprecated. deprecated[6.3.0,
 Use `xpack.monitoring.collection.enabled` set to `false` instead.]
@@ -59,7 +59,7 @@ Controls how often data samples are collected. Defaults to `10s`. If you
 modify the collection interval, set the `xpack.monitoring.min_interval_seconds`
 option in `kibana.yml` to the same value.
 
-`xpack.monitoring.elasticsearch.collection.enabled`:: (<<cluster-update-settings,Dynamic>>) 
+`xpack.monitoring.elasticsearch.collection.enabled` (<<cluster-update-settings,Dynamic>>)::
 
 Controls whether statistics about your {es} cluster should be collected. Defaults to `true`.
 This is different from xpack.monitoring.collection.enabled, which allows you to enable or disable
@@ -71,7 +71,7 @@ to pass through this cluster.
 
 Sets the timeout for collecting the cluster statistics. Defaults to `10s`.
 
-`xpack.monitoring.collection.indices`:: (<<cluster-update-settings,Dynamic>>) 
+`xpack.monitoring.collection.indices` (<<cluster-update-settings,Dynamic>>)::
 
 Controls which indices Monitoring collects data from. Defaults to all indices. Specify the index names
 as a comma-separated list, for example `test1,test2,test3`. Names can include wildcards, for
diff --git a/docs/reference/sql/index.asciidoc b/docs/reference/sql/index.asciidoc
index aa9eebea7b7f6..99b946746cc60 100644
--- a/docs/reference/sql/index.asciidoc
+++ b/docs/reference/sql/index.asciidoc
@@ -3,10 +3,10 @@
 [[xpack-sql]]
 = SQL Access
 
-:sql-tests: {xes-repo-dir}/../../qa/sql
+:sql-tests: {xes-repo-dir}/../../plugin/sql/qa
 :sql-specs: {sql-tests}/src/main/resources
-:jdbc-tests: {sql-tests}/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc
-:security-tests: {sql-tests}/security/src/test/java/org/elasticsearch/xpack/qa/sql/security
+:jdbc-tests: {sql-tests}/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc
+:security-tests: {sql-tests}/security/src/test/java/org/elasticsearch/xpack/sql/qa/security
 :es-sql: Elasticsearch SQL
 
 [partintro]
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/ObjectPath.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectPath.java
similarity index 50%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/ObjectPath.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectPath.java
index 67ef405238aba..8a70f9cb70474 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/ObjectPath.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectPath.java
@@ -1,48 +1,61 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.watcher.support.xcontent;
 
-import org.elasticsearch.common.Strings;
+package org.elasticsearch.common.xcontent;
 
 import java.lang.reflect.Array;
 import java.util.List;
 import java.util.Map;
 
-public class ObjectPath {
+/**
+ * Helper class to navigate nested objects using dot notation
+ */
+public final class ObjectPath {
+
+    private static final String[] EMPTY_ARRAY = new String[0];
 
     private ObjectPath() {
     }
 
+    /**
+     * Return the value within a given object at the specified path, or
+     * {@code null} if the path does not exist
+     */
+    @SuppressWarnings("unchecked")
     public static <T> T eval(String path, Object object) {
         return (T) evalContext(path, object);
     }
 
     private static Object evalContext(String path, Object ctx) {
         final String[] parts;
-        if (path == null || path.isEmpty()) parts = Strings.EMPTY_ARRAY;
+        if (path == null || path.isEmpty()) parts = EMPTY_ARRAY;
         else parts = path.split("\\.");
-        StringBuilder resolved = new StringBuilder();
         for (String part : parts) {
             if (ctx == null) {
                 return null;
             }
             if (ctx instanceof Map) {
                 ctx = ((Map) ctx).get(part);
-                if (resolved.length() != 0) {
-                    resolved.append(".");
-                }
-                resolved.append(part);
             } else if (ctx instanceof List) {
                 try {
                     int index = Integer.parseInt(part);
                     ctx = ((List) ctx).get(index);
-                    if (resolved.length() != 0) {
-                        resolved.append(".");
-                    }
-                    resolved.append(part);
                 } catch (NumberFormatException nfe) {
                     return null;
                 }
@@ -50,10 +63,6 @@ private static Object evalContext(String path, Object ctx) {
                 try {
                     int index = Integer.parseInt(part);
                     ctx = Array.get(ctx, index);
-                    if (resolved.length() != 0) {
-                        resolved.append(".");
-                    }
-                    resolved.append(part);
                 } catch (NumberFormatException nfe) {
                     return null;
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/common/xcontent/XContentUtils.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentUtils.java
similarity index 60%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/common/xcontent/XContentUtils.java
rename to libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentUtils.java
index da8ac3ef9d8f1..14a9f5be24b28 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/common/xcontent/XContentUtils.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentUtils.java
@@ -1,20 +1,34 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.core.watcher.common.xcontent;
 
-import org.elasticsearch.common.xcontent.XContentParser;
+package org.elasticsearch.common.xcontent;
 
 import java.io.IOException;
 
-public class XContentUtils {
+public final class XContentUtils {
 
     private XContentUtils() {
     }
 
-    // TODO open this up in core
+    /**
+     * Convert a {@link XContentParser.Token} to a value
+     */
     public static Object readValue(XContentParser parser, XContentParser.Token token) throws IOException {
         if (token == XContentParser.Token.VALUE_NULL) {
             return null;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/xcontent/MapPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectPathTests.java
similarity index 63%
rename from x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/xcontent/MapPathTests.java
rename to libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectPathTests.java
index f89552a637726..52e9723743b44 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/xcontent/MapPathTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectPathTests.java
@@ -1,12 +1,25 @@
 /*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
  */
-package org.elasticsearch.xpack.watcher.support.xcontent;
+
+package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -18,23 +31,23 @@
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.is;
 
-public class MapPathTests extends ESTestCase {
-    public void testEval() throws Exception {
+public class ObjectPathTests extends ESTestCase {
+    public void testEval() {
         Map<String, Object> map = singletonMap("key", "value");
 
         assertThat(ObjectPath.eval("key", map), is((Object) "value"));
         assertThat(ObjectPath.eval("key1", map), nullValue());
     }
 
-    public void testEvalList() throws Exception {
-        List list = Arrays.asList(1, 2, 3, 4);
+    public void testEvalList() {
+        List<Integer> list = Arrays.asList(1, 2, 3, 4);
         Map<String, Object> map = singletonMap("key", list);
 
         int index = randomInt(3);
         assertThat(ObjectPath.eval("key." + index, map), is(list.get(index)));
     }
 
-    public void testEvalArray() throws Exception {
+    public void testEvalArray() {
         int[] array = new int[] { 1, 2, 3, 4 };
         Map<String, Object> map = singletonMap("key", array);
 
@@ -42,13 +55,13 @@ public void testEvalArray() throws Exception {
         assertThat(((Number) ObjectPath.eval("key." + index, map)).intValue(), is(array[index]));
     }
 
-    public void testEvalMap() throws Exception {
+    public void testEvalMap() {
         Map<String, Object> map = singletonMap("a", singletonMap("b", "val"));
 
         assertThat(ObjectPath.eval("a.b", map), is((Object) "val"));
     }
 
-    public void testEvalMixed() throws Exception {
+    public void testEvalMixed() {
         Map<String, Object> map = new HashMap<>();
 
         Map<String, Object> mapA = new HashMap<>();
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
index 484c9d9b1280b..de99ab530281c 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LegacyDelimitedPayloadTokenFilterFactory.java
@@ -19,21 +19,21 @@
 
 package org.elasticsearch.analysis.common;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
 public class LegacyDelimitedPayloadTokenFilterFactory extends DelimitedPayloadTokenFilterFactory {
-    private static final DeprecationLogger DEPRECATION_LOGGER =
-        new DeprecationLogger(Loggers.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
+    private static final DeprecationLogger deprecationLogger =
+        new DeprecationLogger(LogManager.getLogger(LegacyDelimitedPayloadTokenFilterFactory.class));
 
     LegacyDelimitedPayloadTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, env, name, settings);
         if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_2_0)) {
-            DEPRECATION_LOGGER.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
+            deprecationLogger.deprecated("Deprecated [delimited_payload_filter] used, replaced by [delimited_payload]");
         }
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
index 0698f6ed0a6c9..f4215af8390dd 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
@@ -52,28 +52,30 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
             map = ingestDocument.getSourceAndMetadata();
         }
 
-        if (ingestDocument.hasField(path)) {
-            Object value = map.remove(field);
-            ingestDocument.appendFieldValue(path, value);
-        } else {
-            // check whether we actually can expand the field in question into an object field.
-            // part of the path may already exist and if part of it would be a value field (string, integer etc.)
-            // then we can't override it with an object field and we should fail with a good reason.
-            // IngestDocument#setFieldValue(...) would fail too, but the error isn't very understandable
-            for (int index = path.indexOf('.'); index != -1; index = path.indexOf('.', index + 1)) {
-                String partialPath = path.substring(0, index);
-                if (ingestDocument.hasField(partialPath)) {
-                    Object val = ingestDocument.getFieldValue(partialPath, Object.class);
-                    if ((val instanceof Map) == false) {
-                        throw new IllegalArgumentException("cannot expend [" + path + "], because [" + partialPath +
+        if (map.containsKey(field)) {
+            if (ingestDocument.hasField(path)) {
+                Object value = map.remove(field);
+                ingestDocument.appendFieldValue(path, value);
+            } else {
+                // check whether we actually can expand the field in question into an object field.
+                // part of the path may already exist and if part of it would be a value field (string, integer etc.)
+                // then we can't override it with an object field and we should fail with a good reason.
+                // IngestDocument#setFieldValue(...) would fail too, but the error isn't very understandable
+                for (int index = path.indexOf('.'); index != -1; index = path.indexOf('.', index + 1)) {
+                    String partialPath = path.substring(0, index);
+                    if (ingestDocument.hasField(partialPath)) {
+                        Object val = ingestDocument.getFieldValue(partialPath, Object.class);
+                        if ((val instanceof Map) == false) {
+                            throw new IllegalArgumentException("cannot expend [" + path + "], because [" + partialPath +
                                 "] is not an object field, but a value field");
+                        }
+                    } else {
+                        break;
                     }
-                } else {
-                    break;
                 }
+                Object value = map.remove(field);
+                ingestDocument.setFieldValue(path, value);
             }
-            Object value = map.remove(field);
-            ingestDocument.setFieldValue(path, value);
         }
         return ingestDocument;
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
index fde7f0c9b8a02..d6a207b859eb0 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
@@ -143,4 +143,38 @@ public void testEscapeFields_path() throws Exception {
         assertThat(document.getFieldValue("field.foo.bar.baz", String.class), equalTo("value"));
     }
 
+
+    public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception {
+        //asking to expand a (literal) field that is not present in the source document
+        Map<String, Object> source = new HashMap<>();
+        source.put("foo.bar", "baz1");
+        IngestDocument document = new IngestDocument(source, Collections.emptyMap());
+        //abc.def does not exist in source, so don't mutate document
+        DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "abc.def");
+        processor.execute(document);
+        //hasField returns false since it requires the expanded form, which is not expanded since we did not ask for it to be
+        assertFalse(document.hasField("foo.bar"));
+        //nothing has changed
+        assertEquals(document.getSourceAndMetadata().get("foo.bar"), "baz1");
+        //abc.def is not found anywhere
+        assertFalse(document.hasField("abc.def"));
+        assertFalse(document.getSourceAndMetadata().containsKey("abc"));
+        assertFalse(document.getSourceAndMetadata().containsKey("abc.def"));
+
+        //asking to expand a (literal) field that does not exist, but the nested field does exist
+        source = new HashMap<>();
+        Map<String, Object> inner = new HashMap<>();
+        inner.put("bar", "baz1");
+        source.put("foo", inner);
+        document = new IngestDocument(source, Collections.emptyMap());
+        //foo.bar, the literal value (as opposed to nested value) does not exist in source, so don't mutate document
+        processor = new DotExpanderProcessor("_tag", null, "foo.bar");
+        processor.execute(document);
+        //hasField returns true because the nested/expanded form exists in the source document
+        assertTrue(document.hasField("foo.bar"));
+        //nothing changed
+        assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
+        assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));
+    }
+
 }
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.5.0.jar.sha1
deleted file mode 100644
index 7c391c8fbd1cf..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4566befa0926231cfc86692809bba4f636836a58
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-7.6.0-snapshot-f9598f335b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..ca7f638b808b0
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+717fb4e5f1c1667819465805858731d6683f3ab4
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index fd80c56cdbe8a..cd0b09eca8c3a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -31,6 +31,6 @@ public class ExpressionPlugin extends Plugin implements ScriptPlugin {
 
     @Override
     public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
-        return new ExpressionScriptEngine(settings);
+        return new ExpressionScriptEngine();
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index 6e6d5dc443d3c..b2712a4b19555 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -30,8 +30,6 @@
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -63,14 +61,10 @@
  * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch.  Only
  * {@link SearchScript}s are supported.
  */
-public class ExpressionScriptEngine extends AbstractComponent implements ScriptEngine {
+public class ExpressionScriptEngine implements ScriptEngine {
 
     public static final String NAME = "expression";
 
-    public ExpressionScriptEngine(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
index 33e6239002eb1..9894dd9f4919b 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.script.expression;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@@ -64,7 +63,7 @@ public void setUp() throws Exception {
         when(fieldData.getFieldName()).thenReturn("field");
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
-        service = new ExpressionScriptEngine(Settings.EMPTY);
+        service = new ExpressionScriptEngine();
         lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
index caa9fa4831add..41ac8b720dc03 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java
@@ -46,7 +46,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi
     private int maxConcurrentSearchRequests = 0;
     private List<SearchTemplateRequest> requests = new ArrayList<>();
 
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     /**
      * Add a search template request to execute. Note, the order is important, the search response will be returned in the
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 81cc802916d4e..0ea3d4af81f79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -48,11 +48,6 @@
  */
 final class Compiler {
 
-    /**
-     * The maximum number of characters allowed in the script source.
-     */
-    static final int MAXIMUM_SOURCE_LENGTH = 16384;
-
     /**
      * Define the class with lowest privileges.
      */
@@ -212,12 +207,6 @@ private static void addFactoryMethod(Map<String, Class<?>> additionalClasses, Cl
      * @return An executable script that implements both a specified interface and is a subclass of {@link PainlessScript}
      */
     Constructor<?> compile(Loader loader, MainMethodReserved reserved, String name, String source, CompilerSettings settings) {
-        if (source.length() > MAXIMUM_SOURCE_LENGTH) {
-            throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
-                " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
-                " plugin if a script longer than this length is a requirement.");
-        }
-
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, reserved, name, source, settings, painlessLookup,
                 null);
@@ -248,12 +237,6 @@ Constructor<?> compile(Loader loader, MainMethodReserved reserved, String name,
      * @return The bytes for compilation.
      */
     byte[] compile(String name, String source, CompilerSettings settings, Printer debugStream) {
-        if (source.length() > MAXIMUM_SOURCE_LENGTH) {
-            throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
-                " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
-                " plugin if a script longer than this length is a requirement.");
-        }
-
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SSource root = Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), name, source, settings, painlessLookup,
                 debugStream);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
index 5ed305751c8e8..be753de5721b0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java
@@ -21,7 +21,6 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.painless.Compiler.Loader;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
@@ -57,7 +56,7 @@
 /**
  * Implementation of a ScriptEngine for the Painless language.
  */
-public final class PainlessScriptEngine extends AbstractComponent implements ScriptEngine {
+public final class PainlessScriptEngine implements ScriptEngine {
 
     /**
      * Standard name of the Painless language.
@@ -93,8 +92,6 @@ public final class PainlessScriptEngine extends AbstractComponent implements Scr
      * @param settings The settings to initialize the engine with.
      */
     public PainlessScriptEngine(Settings settings, Map<ScriptContext<?>, List<Whitelist>> contexts) {
-        super(settings);
-
         defaultCompilerSettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(settings));
 
         Map<ScriptContext<?>, Compiler> contextsToCompilers = new HashMap<>();
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
index 6ee021c695f99..4f7dab61df4d3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
@@ -21,16 +21,25 @@
 
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.memory.MemoryIndex;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptedMetricAggContexts;
+import org.elasticsearch.search.lookup.LeafSearchLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public class ScriptedMetricAggContextsTests extends ScriptTestCase {
     @Override
     protected Map<ScriptContext<?>, List<Whitelist>> scriptContexts() {
@@ -58,7 +67,7 @@ public void testInitBasic() {
         assertEquals(10, state.get("testField"));
     }
 
-    public void testMapBasic() {
+    public void testMapBasic() throws IOException {
         ScriptedMetricAggContexts.MapScript.Factory factory = scriptEngine.compile("test",
             "state.testField = 2*_score", ScriptedMetricAggContexts.MapScript.CONTEXT, Collections.emptyMap());
 
@@ -86,6 +95,32 @@ public void testMapBasic() {
         assertEquals(1.0, state.get("testField"));
     }
 
+    public void testMapSourceAccess() throws IOException {
+        ScriptedMetricAggContexts.MapScript.Factory factory = scriptEngine.compile("test",
+            "state.testField = params._source.three", ScriptedMetricAggContexts.MapScript.CONTEXT, Collections.emptyMap());
+
+        Map<String, Object> params = new HashMap<>();
+        Map<String, Object> state = new HashMap<>();
+
+        MemoryIndex index = new MemoryIndex();
+        // we don't need a real index, just need to construct a LeafReaderContext which cannot be mocked
+        LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
+
+        SearchLookup lookup = mock(SearchLookup.class);
+        LeafSearchLookup leafLookup = mock(LeafSearchLookup.class);
+        when(lookup.getLeafSearchLookup(leafReaderContext)).thenReturn(leafLookup);
+        SourceLookup sourceLookup = mock(SourceLookup.class);
+        when(leafLookup.asMap()).thenReturn(Collections.singletonMap("_source", sourceLookup));
+        when(sourceLookup.get("three")).thenReturn(3);
+        ScriptedMetricAggContexts.MapScript.LeafFactory leafFactory = factory.newFactory(params, state, lookup);
+        ScriptedMetricAggContexts.MapScript script = leafFactory.newInstance(leafReaderContext);
+
+        script.execute();
+
+        assert(state.containsKey("testField"));
+        assertEquals(3, state.get("testField"));
+    }
+
     public void testCombineBasic() {
         ScriptedMetricAggContexts.CombineScript.Factory factory = scriptEngine.compile("test",
             "state.testField = params.initialVal; return state.testField + params.inc", ScriptedMetricAggContexts.CombineScript.CONTEXT,
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 32d74d0837ccd..d1db6606c86aa 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.script.ScriptException;
 
 import java.lang.invoke.WrongMethodTypeException;
-import java.util.Arrays;
 import java.util.Collections;
 
 import static java.util.Collections.emptyMap;
@@ -200,21 +199,6 @@ public void testLoopLimits() {
                    "The maximum number of statements that can be executed in a loop has been reached."));
     }
 
-    public void testSourceLimits() {
-        final char[] tooManyChars = new char[Compiler.MAXIMUM_SOURCE_LENGTH + 1];
-        Arrays.fill(tooManyChars, '0');
-
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, false, () -> {
-            exec(new String(tooManyChars));
-        });
-        assertTrue(expected.getMessage().contains("Scripts may be no longer than"));
-
-        final char[] exactlyAtLimit = new char[Compiler.MAXIMUM_SOURCE_LENGTH];
-        Arrays.fill(exactlyAtLimit, '0');
-        // ok
-        assertEquals(0, exec(new String(exactlyAtLimit)));
-    }
-
     public void testIllegalDynamicMethod() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'test'; return x.getClass().toString()");
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml
index b7be116b38695..91d539de73c7f 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml
@@ -23,8 +23,8 @@
         indices.refresh: {}
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     script:
@@ -45,8 +45,8 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     script:
@@ -70,8 +70,8 @@
     - match: { hits.hits.1.fields.sNum1.0: 3.0 }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     script:
@@ -96,8 +96,8 @@
     - match: { hits.hits.2.fields.sNum1.0: 3.0 }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     script:
@@ -127,8 +127,8 @@
         indices.refresh: {}
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -149,8 +149,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -171,8 +171,8 @@
     - match: { hits.hits.1._id: "2" }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -193,8 +193,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -215,8 +215,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -237,8 +237,8 @@
     - match: { hits.hits.1._id: "1" }
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -274,8 +274,8 @@
         indices.refresh: {}
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -325,8 +325,8 @@
 
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     function_score:
@@ -364,8 +364,8 @@
 
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 script_fields:
                     foobar:
@@ -391,8 +391,8 @@
 
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 aggs:
                     value_agg:
@@ -428,8 +428,8 @@
 
     - do:
         catch: bad_request
-        index: test
         search:
+            index: test
             body:
                 aggs:
                     genre:
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
index 70d1c6169d0a6..22971f1484f57 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
@@ -75,7 +75,7 @@ public void testDefaults() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         assertEquals(1230, pointField.numericValue().longValue());
         IndexableField dvField = fields[1];
@@ -149,7 +149,7 @@ public void testNoDocValues() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(1230, pointField.numericValue().longValue());
     }
 
@@ -173,7 +173,7 @@ public void testStore() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(3, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(1230, pointField.numericValue().doubleValue(), 0d);
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
@@ -202,7 +202,7 @@ public void testCoerce() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(1230, pointField.numericValue().longValue());
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
@@ -317,7 +317,7 @@ public void testNullValue() throws IOException {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         assertEquals(25, pointField.numericValue().longValue());
         IndexableField dvField = fields[1];
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 23839a3a635ab..24b333bed6643 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -341,7 +341,7 @@ Tuple<List<BytesRef>, Map<String, List<byte[]>>> extractTermsAndRanges(IndexRead
                         extractedTerms.add(builder.toBytesRef());
                     }
                 }
-                if (info.getPointDimensionCount() == 1) { // not != 0 because range fields are not supported
+                if (info.getPointIndexDimensionCount() == 1) { // not != 0 because range fields are not supported
                     PointValues values = reader.getPointValues(info.name);
                     List<byte[]> encodedPointValues = new ArrayList<>();
                     encodedPointValues.add(values.getMinPackedValue().clone());
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 44823f9aa012b..0338e0fba91df 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.SearchHit;
@@ -56,8 +55,8 @@
 final class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
     private final HighlightPhase highlightPhase;
 
-    PercolatorHighlightSubFetchPhase(Settings settings, Map<String, Highlighter> highlighters) {
-        this.highlightPhase = new HighlightPhase(settings, highlighters);
+    PercolatorHighlightSubFetchPhase(Map<String, Highlighter> highlighters) {
+        this.highlightPhase = new HighlightPhase(highlighters);
     }
 
     boolean hitsExecutionNeeded(SearchContext context) { // for testing
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 7128060448cf1..659b9c422fc0e 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.percolator;
 
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -35,13 +34,6 @@
 import static java.util.Collections.singletonMap;
 
 public class PercolatorPlugin extends Plugin implements MapperPlugin, SearchPlugin {
-
-    private final Settings settings;
-
-    public PercolatorPlugin(Settings settings) {
-        this.settings = settings;
-    }
-
     @Override
     public List<QuerySpec<?>> getQueries() {
         return singletonList(new QuerySpec<>(PercolateQueryBuilder.NAME, PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent));
@@ -51,7 +43,7 @@ public List<QuerySpec<?>> getQueries() {
     public List<FetchSubPhase> getFetchSubPhases(FetchPhaseConstructionContext context) {
         return Arrays.asList(
             new PercolatorMatchedSlotSubFetchPhase(),
-            new PercolatorHighlightSubFetchPhase(settings, context.getHighlighters())
+            new PercolatorHighlightSubFetchPhase(context.getHighlighters())
         );
     }
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index f1b89d92ab11e..e49204506e588 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
@@ -47,8 +46,7 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
     public void testHitsExecutionNeeded() {
         PercolateQuery percolateQuery = new PercolateQuery("_name", ctx -> null, Collections.singletonList(new BytesArray("{}")),
             new MatchAllDocsQuery(), Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery());
-        PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(Settings.EMPTY,
-            emptyMap());
+        PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(emptyMap());
         SearchContext searchContext = Mockito.mock(SearchContext.class);
         Mockito.when(searchContext.highlight()).thenReturn(new SearchContextHighlight(Collections.emptyList()));
         Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery());
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
index 7d594c852da5b..5c98cce9c8e59 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -286,7 +286,7 @@ public void testIndicesOptions() {
         // test that ignore_unavailable=true works but returns one result less
         assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
 
-        request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(6, details.getRetrieved());
@@ -294,37 +294,37 @@ public void testIndicesOptions() {
 
         // test that ignore_unavailable=false or default settings throw an IndexClosedException
         assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged());
-        request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
 
         // test expand_wildcards
         request = new RankEvalRequest(task, new String[] { "tes*" });
-        request.indicesOptions(IndicesOptions.fromParameters("none", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(0, details.getRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters("open", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(6, details.getRetrieved());
         assertEquals(5, details.getRelevantRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
 
         // test allow_no_indices
         request = new RankEvalRequest(task, new String[] { "bad*" });
-        request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails();
         assertEquals(0, details.getRetrieved());
 
-        request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
+        request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
         assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class));
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java
index 10e3611b30d31..1a16c311fcf3a 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java
@@ -59,7 +59,8 @@ protected RankEvalRequest createTestInstance() {
         }
         RankEvalRequest rankEvalRequest = new RankEvalRequest(RankEvalSpecTests.createTestItem(), indices);
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+            randomBoolean());
         rankEvalRequest.indicesOptions(indicesOptions);
         return rankEvalRequest;
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
index 1a54f53749fb2..e2f935125ecbd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
@@ -23,73 +23,133 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequestBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
+import org.elasticsearch.index.reindex.CancelTests;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.index.reindex.ReindexPlugin;
+import org.elasticsearch.index.reindex.ReindexRequestBuilder;
 import org.elasticsearch.index.reindex.RethrottleAction;
 import org.elasticsearch.index.reindex.UpdateByQueryAction;
 import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder;
+import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+import org.junit.Before;
 
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
 
 public class ReindexDocumentationIT extends ESIntegTestCase {
 
-    public void reindex() {
+    // Semaphore used to allow & block indexing operations during the test
+    private static final Semaphore ALLOWED_OPERATIONS = new Semaphore(0);
+    private static final String INDEX_NAME = "source_index";
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(ReindexPlugin.class, ReindexCancellationPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singletonList(ReindexPlugin.class);
+    }
+
+    @Before
+    public void setup() {
+        client().admin().indices().prepareCreate(INDEX_NAME).get();
+    }
+
+    public void testReindex() {
         Client client = client();
         // tag::reindex1
-        BulkByScrollResponse response = ReindexAction.INSTANCE.newRequestBuilder(client)
+        BulkByScrollResponse response =
+            ReindexAction.INSTANCE.newRequestBuilder(client)
+            .source("source_index")
             .destination("target_index")
             .filter(QueryBuilders.matchQuery("category", "xzy")) // <1>
             .get();
         // end::reindex1
     }
 
-    public void updateByQuery() {
+    public void testUpdateByQuery() {
         Client client = client();
+        client.admin().indices().prepareCreate("foo").get();
+        client.admin().indices().prepareCreate("bar").get();
+        client.admin().indices().preparePutMapping(INDEX_NAME).setType("_doc").setSource("cat", "type=keyword").get();
         {
             // tag::update-by-query
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source("source_index").abortOnVersionConflict(false);
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query
         }
         {
             // tag::update-by-query-filter
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source("source_index")
                 .filter(QueryBuilders.termQuery("level", "awesome"))
                 .size(1000)
-                .script(new Script(ScriptType.INLINE, "ctx._source.awesome = 'absolutely'", "painless", Collections.emptyMap()));
+                .script(new Script(ScriptType.INLINE,
+                    "ctx._source.awesome = 'absolutely'",
+                    "painless",
+                    Collections.emptyMap()));
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-filter
         }
         {
             // tag::update-by-query-size
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source("source_index")
-                .source().setSize(500);
+                .source()
+                .setSize(500);
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-size
         }
         {
             // tag::update-by-query-sort
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
-            updateByQuery.source("source_index").size(100)
-                .source().addSort("cat", SortOrder.DESC);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            updateByQuery.source("source_index")
+                .size(100)
+                .source()
+                .addSort("cat", SortOrder.DESC);
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-sort
         }
         {
             // tag::update-by-query-script
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source("source_index")
                 .script(new Script(
                     ScriptType.INLINE,
@@ -106,53 +166,67 @@ public void updateByQuery() {
         }
         {
             // tag::update-by-query-multi-index
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source("foo", "bar").source().setTypes("a", "b");
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-multi-index
         }
         {
             // tag::update-by-query-routing
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.source().setRouting("cat");
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-routing
         }
         {
             // tag::update-by-query-pipeline
-            UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+            UpdateByQueryRequestBuilder updateByQuery =
+              UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
             updateByQuery.setPipeline("hurray");
             BulkByScrollResponse response = updateByQuery.get();
             // end::update-by-query-pipeline
         }
+    }
+
+    public void testTasks() throws Exception {
+        final Client client = client();
+        final ReindexRequestBuilder builder = reindexAndPartiallyBlock();
+
         {
             // tag::update-by-query-list-tasks
             ListTasksResponse tasksList = client.admin().cluster().prepareListTasks()
                 .setActions(UpdateByQueryAction.NAME).setDetailed(true).get();
             for (TaskInfo info: tasksList.getTasks()) {
                 TaskId taskId = info.getTaskId();
-                BulkByScrollTask.Status status = (BulkByScrollTask.Status) info.getStatus();
+                BulkByScrollTask.Status status =
+                    (BulkByScrollTask.Status) info.getStatus();
                 // do stuff
             }
             // end::update-by-query-list-tasks
         }
+
+        TaskInfo mainTask = CancelTests.findTaskToCancel(ReindexAction.NAME, builder.request().getSlices());
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus();
+        assertNull(status.getReasonCancelled());
+        TaskId taskId = mainTask.getTaskId();
         {
-            TaskId taskId = null;
             // tag::update-by-query-get-task
             GetTaskResponse get = client.admin().cluster().prepareGetTask(taskId).get();
             // end::update-by-query-get-task
         }
         {
-            TaskId taskId = null;
             // tag::update-by-query-cancel-task
             // Cancel all update-by-query requests
-            client.admin().cluster().prepareCancelTasks().setActions(UpdateByQueryAction.NAME).get().getTasks();
+            client.admin().cluster().prepareCancelTasks()
+                .setActions(UpdateByQueryAction.NAME).get().getTasks();
             // Cancel a specific update-by-query request
-            client.admin().cluster().prepareCancelTasks().setTaskId(taskId).get().getTasks();
+            client.admin().cluster().prepareCancelTasks()
+                .setTaskId(taskId).get().getTasks();
             // end::update-by-query-cancel-task
         }
         {
-            TaskId taskId = null;
             // tag::update-by-query-rethrottle
             RethrottleAction.INSTANCE.newRequestBuilder(client)
                 .setTaskId(taskId)
@@ -160,12 +234,18 @@ public void updateByQuery() {
                 .get();
             // end::update-by-query-rethrottle
         }
+
+        // unblocking the blocked update
+        ALLOWED_OPERATIONS.release(builder.request().getSlices());
     }
 
-    public void deleteByQuery() {
+    public void testDeleteByQuery() {
         Client client = client();
+        client.admin().indices().prepareCreate("persons").get();
+
         // tag::delete-by-query-sync
-        BulkByScrollResponse response = DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
+        BulkByScrollResponse response =
+          DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
             .filter(QueryBuilders.matchQuery("gender", "male")) // <1>
             .source("persons")                                  // <2>
             .get();                                             // <3>
@@ -189,4 +269,76 @@ public void onFailure(Exception e) {
         // end::delete-by-query-async
     }
 
+    /**
+     * Similar to what CancelTests does: blocks some operations to be able to catch some tasks in running state
+     * @see CancelTests#testCancel(String, AbstractBulkByScrollRequestBuilder, CancelTests.CancelAssertion, Matcher)
+     */
+    private ReindexRequestBuilder reindexAndPartiallyBlock() throws Exception {
+        final Client client = client();
+        final int numDocs = randomIntBetween(10, 100);
+        ALLOWED_OPERATIONS.release(numDocs);
+
+        indexRandom(true, false, true, IntStream.range(0, numDocs)
+            .mapToObj(i -> client().prepareIndex(INDEX_NAME, "_doc", Integer.toString(i)).setSource("n", Integer.toString(i)))
+            .collect(Collectors.toList()));
+
+        // Checks that the all documents have been indexed and correctly counted
+        assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numDocs);
+        assertThat(ALLOWED_OPERATIONS.drainPermits(), equalTo(0));
+
+        ReindexRequestBuilder builder = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source(INDEX_NAME)
+            .destination("target_index", "_doc");
+        // Scroll by 1 so that cancellation is easier to control
+        builder.source().setSize(1);
+
+        int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs);
+        // chose to modify some of docs - rest is still blocked
+        ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices());
+
+        // Now execute the reindex action...
+        builder.execute();
+
+        // 10 seconds is usually fine but on heavily loaded machines this can take a while
+        assertTrue("updates blocked", awaitBusy(
+            () -> ALLOWED_OPERATIONS.hasQueuedThreads() && ALLOWED_OPERATIONS.availablePermits() == 0,
+            1, TimeUnit.MINUTES));
+        return builder;
+    }
+
+    public static class ReindexCancellationPlugin extends Plugin {
+
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            indexModule.addIndexOperationListener(new BlockingOperationListener());
+        }
+    }
+
+    public static class BlockingOperationListener implements IndexingOperationListener {
+
+        @Override
+        public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
+            return preCheck(index, index.type());
+        }
+
+        @Override
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
+            return preCheck(delete, delete.type());
+        }
+
+        private <T extends Engine.Operation> T preCheck(T operation, String type) {
+            if (("_doc".equals(type) == false) || (operation.origin() != Engine.Operation.Origin.PRIMARY)) {
+                return operation;
+            }
+
+            try {
+                if (ALLOWED_OPERATIONS.tryAcquire(30, TimeUnit.SECONDS)) {
+                    return operation;
+                }
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            throw new IllegalStateException("Something went wrong");
+        }
+    }
+
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 6b7b21a55148d..6d6ae01f0626c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
@@ -26,7 +27,6 @@
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.engine.Engine;
@@ -195,7 +195,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder<?, ?>
         assertion.assertThat(response, numDocs, numModifiedDocs);
     }
 
-    private TaskInfo findTaskToCancel(String actionName, int workerCount) {
+    public static TaskInfo findTaskToCancel(String actionName, int workerCount) {
         ListTasksResponse tasks;
         long start = System.nanoTime();
         do {
@@ -298,7 +298,7 @@ public void onIndexModule(IndexModule indexModule) {
     }
 
     public static class BlockingOperationListener implements IndexingOperationListener {
-        private static final Logger log = Loggers.getLogger(CancelTests.class);
+        private static final Logger log = LogManager.getLogger(CancelTests.class);
 
         @Override
         public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
index 158ecff9b2b4e..a7042b8bfee2b 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.BlobStoreException;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -34,7 +33,7 @@
 /**
  * Read-only URL-based blob store
  */
-public class URLBlobStore extends AbstractComponent implements BlobStore {
+public class URLBlobStore implements BlobStore {
 
     private final URL path;
 
@@ -53,7 +52,6 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
      * @param path     base URL
      */
     public URLBlobStore(Settings settings, URL path) {
-        super(settings);
         this.path = path;
         this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size",
             new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
index 98b8c0a1945a5..8f8ae805fd1e8 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
@@ -82,21 +82,21 @@ public URLRepository(RepositoryMetaData metadata, Environment environment,
                          NamedXContentRegistry namedXContentRegistry) {
         super(metadata, environment.settings(), namedXContentRegistry);
 
-        if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
+        if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) ==  false) {
             throw new RepositoryException(metadata.name(), "missing url");
         }
         this.environment = environment;
-        supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
-        urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
+        supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(environment.settings());
+        urlWhiteList = ALLOWED_URLS_SETTING.get(environment.settings()).toArray(new URIPattern[]{});
         basePath = BlobPath.cleanPath();
         url = URL_SETTING.exists(metadata.settings())
-            ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
+            ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(environment.settings());
     }
 
     @Override
     protected BlobStore createBlobStore() {
         URL normalizedURL = checkURL(url);
-        return new URLBlobStore(settings, normalizedURL);
+        return new URLBlobStore(environment.settings(), normalizedURL);
     }
 
     // only use for testing
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index 71e3b42279ca4..5373f09a48d92 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -193,7 +193,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE,
             Property.NodeScope, Property.Deprecated);
 
-
+    private final Settings settings;
     protected final NetworkService networkService;
     protected final BigArrays bigArrays;
 
@@ -254,6 +254,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
                                      NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
         super(settings);
+        this.settings = settings;
         Netty4Utils.setAvailableProcessors(EsExecutors.PROCESSORS_SETTING.get(settings));
         this.networkService = networkService;
         this.bigArrays = bigArrays;
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
index d4b3aead0d2fd..38527151695d8 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport.netty4;
 
 import io.netty.util.internal.logging.AbstractInternalLogger;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.common.SuppressLoggerChecks;
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index c8b96b8bf6da2..a6104bed48af8 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -36,13 +36,11 @@
 import io.netty.util.AttributeKey;
 import io.netty.util.concurrent.Future;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
@@ -59,8 +57,6 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
@@ -101,8 +97,9 @@ public class Netty4Transport extends TcpTransport {
     private final int workerCount;
     private final ByteSizeValue receivePredictorMin;
     private final ByteSizeValue receivePredictorMax;
-    private volatile Bootstrap clientBootstrap;
     private final Map<String, ServerBootstrap> serverBootstraps = newConcurrentMap();
+    private volatile Bootstrap clientBootstrap;
+    private volatile NioEventLoopGroup eventLoopGroup;
 
     public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                            NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
@@ -125,10 +122,12 @@ public Netty4Transport(Settings settings, ThreadPool threadPool, NetworkService
     protected void doStart() {
         boolean success = false;
         try {
-            clientBootstrap = createClientBootstrap();
+            ThreadFactory threadFactory = daemonThreadFactory(settings, TRANSPORT_WORKER_THREAD_NAME_PREFIX);
+            eventLoopGroup = new NioEventLoopGroup(workerCount, threadFactory);
+            clientBootstrap = createClientBootstrap(eventLoopGroup);
             if (NetworkService.NETWORK_SERVER.get(settings)) {
                 for (ProfileSettings profileSettings : profileSettings) {
-                    createServerBootstrap(profileSettings);
+                    createServerBootstrap(profileSettings, eventLoopGroup);
                     bindServer(profileSettings);
                 }
             }
@@ -141,9 +140,9 @@ protected void doStart() {
         }
     }
 
-    private Bootstrap createClientBootstrap() {
+    private Bootstrap createClientBootstrap(NioEventLoopGroup eventLoopGroup) {
         final Bootstrap bootstrap = new Bootstrap();
-        bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)));
+        bootstrap.group(eventLoopGroup);
         bootstrap.channel(NioSocketChannel.class);
 
         bootstrap.option(ChannelOption.TCP_NODELAY, TCP_NO_DELAY.get(settings));
@@ -167,7 +166,7 @@ private Bootstrap createClientBootstrap() {
         return bootstrap;
     }
 
-    private void createServerBootstrap(ProfileSettings profileSettings) {
+    private void createServerBootstrap(ProfileSettings profileSettings, NioEventLoopGroup eventLoopGroup) {
         String name = profileSettings.profileName;
         if (logger.isDebugEnabled()) {
             logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], "
@@ -176,12 +175,9 @@ private void createServerBootstrap(ProfileSettings profileSettings) {
                 receivePredictorMin, receivePredictorMax);
         }
 
-
-        final ThreadFactory workerFactory = daemonThreadFactory(this.settings, TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX, name);
-
         final ServerBootstrap serverBootstrap = new ServerBootstrap();
 
-        serverBootstrap.group(new NioEventLoopGroup(workerCount, workerFactory));
+        serverBootstrap.group(eventLoopGroup);
         serverBootstrap.channel(NioServerSocketChannel.class);
 
         serverBootstrap.childHandler(getServerChannelInitializer(name));
@@ -279,25 +275,14 @@ long failedPingCount() {
     @SuppressForbidden(reason = "debug")
     protected void stopInternal() {
         Releasables.close(() -> {
-            final List<Tuple<String, Future<?>>> serverBootstrapCloseFutures = new ArrayList<>(serverBootstraps.size());
-            for (final Map.Entry<String, ServerBootstrap> entry : serverBootstraps.entrySet()) {
-                serverBootstrapCloseFutures.add(
-                    Tuple.tuple(entry.getKey(), entry.getValue().config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS)));
+            Future<?> shutdownFuture = eventLoopGroup.shutdownGracefully(0, 5, TimeUnit.SECONDS);
+            shutdownFuture.awaitUninterruptibly();
+            if (shutdownFuture.isSuccess() == false) {
+                logger.warn("Error closing netty event loop group", shutdownFuture.cause());
             }
-            for (final Tuple<String, Future<?>> future : serverBootstrapCloseFutures) {
-                future.v2().awaitUninterruptibly();
-                if (!future.v2().isSuccess()) {
-                    logger.debug(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                            "Error closing server bootstrap for profile [{}]", future.v1()), future.v2().cause());
-                }
-            }
-            serverBootstraps.clear();
 
-            if (clientBootstrap != null) {
-                clientBootstrap.config().group().shutdownGracefully(0, 5, TimeUnit.SECONDS).awaitUninterruptibly();
-                clientBootstrap = null;
-            }
+            serverBootstraps.clear();
+            clientBootstrap = null;
         });
     }
 
diff --git a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java
index d8e88e51e2232..6ffd7a941c385 100644
--- a/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java
+++ b/modules/tribe/src/main/java/org/elasticsearch/tribe/TribeDiscovery.java
@@ -45,10 +45,13 @@
  */
 public class TribeDiscovery extends SingleNodeDiscovery implements Discovery {
 
+    private final Settings settings;
+
     @Inject
     public TribeDiscovery(Settings settings, TransportService transportService,
                           MasterService masterService, ClusterApplier clusterApplier) {
         super(settings, transportService, masterService, clusterApplier);
+        this.settings = settings;
     }
 
     @Override
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0.jar.sha1
deleted file mode 100644
index fca8c55d842be..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d156361604898e3d9e5c751c9308b7b856eec523
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..ca19e2c37d9eb
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+4eaf937aa95487a518d458666da78b68fff35ef4
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0.jar.sha1
deleted file mode 100644
index dd14f717dba43..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d412659d8b0dd6a579888cab192fe86d468953f3
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..29593925f9050
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+cdb7c83e29aa6c365e2c06c843ddfe16cfba12cc
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0.jar.sha1
deleted file mode 100644
index 0987a33f65af2..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9fdfafd182aa18dc42854f595bed9c65319786a5
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..c43883bd34046
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+00cb9ce94be5f24bf6e35e4a8cfe641b01c37cf3
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0.jar.sha1
deleted file mode 100644
index 5ccd99775fb57..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-708120bc7ce6d6bde1a4a30b6bb7edeee1fc17ed
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..cb2d33a265c5f
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+5fa1fa9c15ff4646dca8bf68696cfb4f85adbe15
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0.jar.sha1
deleted file mode 100644
index 78d18568a8f43..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ec339902ac6c05440340732b992dd3b73d66d899
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..f5adb601ba30a
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+54790d0478d6411ae9a237b4d29056030e48b87e
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0.jar.sha1
deleted file mode 100644
index 94fa78a5da2a8..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a32e24d0781b97b1f2a3e86268d188180f4e41cb
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..cb9b2365db7fb
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+d240eff790c236180562f20b697db2915d0100cf
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0.jar.sha1
deleted file mode 100644
index fdbbc34edbba4..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ceafdf4a0d8cc9d61110c9ac6ba610e8485620b9
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.6.0-snapshot-f9598f335b.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..dfde53098f73b
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+6bde728277806062a9473626c312a52c52b6f72a
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
index 1a9265de2a72f..987942ef4f031 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
@@ -93,6 +93,7 @@ public static Deployment fromString(String string) {
         }
     }
 
+    private final Settings settings;
     private final AzureComputeService azureComputeService;
     private TransportService transportService;
     private NetworkService networkService;
@@ -107,7 +108,7 @@ public static Deployment fromString(String string) {
 
     public AzureUnicastHostsProvider(Settings settings, AzureComputeService azureComputeService,
                                      TransportService transportService, NetworkService networkService) {
-        super(settings);
+        this.settings = settings;
         this.azureComputeService = azureComputeService;
         this.transportService = transportService;
         this.networkService = networkService;
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
index 4255fc210fb6c..795db2846cef3 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.plugin.discovery.azure.classic;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceImpl;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -43,7 +43,7 @@ public class AzureDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
 
     public static final String AZURE = "azure";
     protected final Settings settings;
-    private static final Logger logger = Loggers.getLogger(AzureDiscoveryPlugin.class);
+    private static final Logger logger = LogManager.getLogger(AzureDiscoveryPlugin.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     public AzureDiscoveryPlugin(Settings settings) {
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index c5a5f1df98ee2..35c2e7336a7d6 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -24,10 +24,10 @@
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpsConfigurator;
 import com.sun.net.httpserver.HttpsServer;
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -243,7 +243,7 @@ public static void startHttpd() throws Exception {
                 responseBody.write(responseAsBytes);
                 responseBody.close();
             } catch (XMLStreamException e) {
-                Loggers.getLogger(AzureDiscoveryClusterFormationTests.class).error("Failed serializing XML", e);
+                LogManager.getLogger(AzureDiscoveryClusterFormationTests.class).error("Failed serializing XML", e);
                 throw new RuntimeException(e);
             }
         });
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index e32ba6948d62d..b08561ffde0ed 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -48,9 +48,30 @@ bundlePlugin {
   }
 }
 
+task writeTestJavaPolicy {
+  doLast {
+    final File tmp = file("${buildDir}/tmp")
+    if (tmp.exists() == false && tmp.mkdirs() == false) {
+      throw new GradleException("failed to create temporary directory [${tmp}]")
+    }
+    final File javaPolicy = file("${tmp}/java.policy")
+    javaPolicy.write(
+    [
+            "grant {",
+            "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+            "};"
+    ].join("\n"))
+  }
+}
+
 test {
+  dependsOn writeTestJavaPolicy
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name
+  
+  // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
+  // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
+  systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
 }
 
 check {
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
index 8e0962db6037c..5caf7240a0ffc 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
+++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
@@ -55,7 +55,10 @@ integTestCluster {
     keystoreSetting 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
     keystoreSetting 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
     setting 'discovery.zen.hosts_provider', 'ec2'
+    setting 'network.host', '_ec2_'
     setting 'discovery.ec2.endpoint', "http://${-> ec2Fixture.addressAndPort}"
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> ec2Fixture.addressAndPort}"
+    
     unicastTransportUri = { seedNode, node, ant -> return null }
 
     waitCondition = { node, ant ->
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
index 0cf4cbdeadb34..6027bd861590e 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
+++ b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java
@@ -19,6 +19,8 @@
 package org.elasticsearch.discovery.ec2;
 
 import org.apache.http.NameValuePair;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.rest.RestStatus;
@@ -60,7 +62,7 @@ public static void main(String[] args) throws Exception {
 
     @Override
     protected Response handle(final Request request) throws IOException {
-        if ("/".equals(request.getPath()) && ("POST".equals(request.getMethod()))) {
+        if ("/".equals(request.getPath()) && (HttpPost.METHOD_NAME.equals(request.getMethod()))) {
             final String userAgent = request.getHeader("User-Agent");
             if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
                 // Simulate an EC2 DescribeInstancesResponse
@@ -74,6 +76,9 @@ protected Response handle(final Request request) throws IOException {
                 return new Response(RestStatus.OK.getStatus(), contentType("text/xml; charset=UTF-8"), responseBody);
             }
         }
+        if ("/latest/meta-data/local-ipv4".equals(request.getPath()) && (HttpGet.METHOD_NAME.equals(request.getMethod()))) {
+            return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, "127.0.0.1".getBytes(UTF_8));
+        }
         return null;
     }
 
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
index a65500d9e2289..6d677d03c3909 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.LazyInitializable;
 
 import java.util.Random;
@@ -41,15 +40,9 @@
 
 class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service {
 
-    public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
-
     private final AtomicReference<LazyInitializable<AmazonEc2Reference, ElasticsearchException>> lazyClientReference =
             new AtomicReference<>();
 
-    AwsEc2ServiceImpl(Settings settings) {
-        super(settings);
-    }
-
     private AmazonEC2 buildClient(Ec2ClientSettings clientSettings) {
         final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings);
         final ClientConfiguration configuration = buildConfiguration(logger, clientSettings);
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
index 8f5037042986b..2817c1c3b60bf 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
@@ -69,7 +69,6 @@ class AwsEc2UnicastHostsProvider extends AbstractComponent implements UnicastHos
     private final TransportAddressesCache dynamicHosts;
 
     AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service) {
-        super(settings);
         this.transportService = transportService;
         this.awsEc2Service = awsEc2Service;
 
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java
index d76c9e820b8b1..650e2747641e2 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java
@@ -24,9 +24,9 @@
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.BasicSessionCredentials;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
@@ -75,7 +75,7 @@ final class Ec2ClientSettings {
     static final Setting<TimeValue> READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout",
             TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope);
 
-    private static final Logger logger = Loggers.getLogger(Ec2ClientSettings.class);
+    private static final Logger logger = LogManager.getLogger(Ec2ClientSettings.class);
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
 
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
index d476d00eef860..c35810774641a 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -19,11 +19,12 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.util.EC2MetadataUtils;
 import com.amazonaws.util.json.Jackson;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -51,9 +52,7 @@
 import java.util.function.Supplier;
 
 public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, ReloadablePlugin {
-
-    private static Logger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
-
+    private static Logger logger = LogManager.getLogger(Ec2DiscoveryPlugin.class);
     public static final String EC2 = "ec2";
 
     static {
@@ -79,7 +78,7 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Reloa
     protected final AwsEc2Service ec2Service;
 
     public Ec2DiscoveryPlugin(Settings settings) {
-        this(settings, new AwsEc2ServiceImpl(settings));
+        this(settings, new AwsEc2ServiceImpl());
     }
 
     protected Ec2DiscoveryPlugin(Settings settings, AwsEc2ServiceImpl ec2Service) {
@@ -92,7 +91,7 @@ protected Ec2DiscoveryPlugin(Settings settings, AwsEc2ServiceImpl ec2Service) {
     @Override
     public NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
         logger.debug("Register _ec2_, _ec2:xxx_ network names");
-        return new Ec2NameResolver(settings);
+        return new Ec2NameResolver();
     }
 
     @Override
@@ -130,7 +129,8 @@ public Settings additionalSettings() {
         final Settings.Builder builder = Settings.builder();
 
         // Adds a node attribute for the ec2 availability zone
-        final String azMetadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone";
+        final String azMetadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService()
+            + "/latest/meta-data/placement/availability-zone";
         builder.put(getAvailabilityZoneNodeAttributes(settings, azMetadataUrl));
         return builder.build();
     }
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
index 92bd01dd9aec7..e9dd3a10e4cc3 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.amazonaws.util.EC2MetadataUtils;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
-import org.elasticsearch.common.settings.Settings;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -79,13 +79,6 @@ private enum Ec2HostnameType {
         }
     }
 
-    /**
-     * Construct a {@link CustomNameResolver}.
-     */
-    Ec2NameResolver(Settings settings) {
-        super(settings);
-    }
-
     /**
      * @param type the ec2 hostname type to discover.
      * @return the appropriate host resolved from ec2 meta-data, or null if it cannot be obtained.
@@ -94,7 +87,7 @@ private enum Ec2HostnameType {
     @SuppressForbidden(reason = "We call getInputStream in doPrivileged and provide SocketPermission")
     public InetAddress[] resolve(Ec2HostnameType type) throws IOException {
         InputStream in = null;
-        String metadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + type.ec2Name;
+        String metadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/meta-data/" + type.ec2Name;
         try {
             URL url = new URL(metadataUrl);
             logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url);
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
index 0596dd697b2eb..e44087f941349 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
@@ -24,8 +24,6 @@
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.Tag;
 
-import org.elasticsearch.common.settings.Settings;
-
 import java.util.List;
 
 public class AwsEc2ServiceMock extends AwsEc2ServiceImpl {
@@ -33,8 +31,7 @@ public class AwsEc2ServiceMock extends AwsEc2ServiceImpl {
     private final int nodes;
     private final List<List<Tag>> tagsList;
 
-    public AwsEc2ServiceMock(Settings settings, int nodes, List<List<Tag>> tagsList) {
-        super(settings);
+    public AwsEc2ServiceMock(int nodes, List<List<Tag>> tagsList) {
         this.nodes = nodes;
         this.tagsList = tagsList;
     }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginMock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginMock.java
index a92bd243bc9b7..bc45a95c2f309 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginMock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginMock.java
@@ -32,7 +32,7 @@ public class Ec2DiscoveryPluginMock extends Ec2DiscoveryPlugin {
     }
 
     public Ec2DiscoveryPluginMock(Settings settings, int nodes, List<List<Tag>> tagsList) {
-        super(settings, new AwsEc2ServiceMock(settings, nodes, tagsList));
+        super(settings, new AwsEc2ServiceMock(nodes, tagsList));
     }
 
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index 295df0c818a91..aa619409c16eb 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -298,7 +298,7 @@ abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider {
     }
 
     public void testGetNodeListEmptyCache() throws Exception {
-        AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(Settings.EMPTY, 1, null);
+        AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(1, null);
         DummyEc2HostProvider provider = new DummyEc2HostProvider(Settings.EMPTY, transportService, awsEc2Service) {
             @Override
             protected List<TransportAddress> fetchDynamicNodes() {
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
index 52bf7e67b0d7d..dedf56b836eb3 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java
@@ -19,35 +19,98 @@
 
 package org.elasticsearch.discovery.ec2;
 
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.function.BiConsumer;
 
+import static com.amazonaws.SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Test for EC2 network.host settings.
+ * <p>
+ * Warning: This test doesn't assert that the exceptions are thrown.
+ * They aren't.
  */
+@SuppressForbidden(reason = "use http server")
 public class Ec2NetworkTests extends ESTestCase {
+
+    private static HttpServer httpServer;
+
+    @BeforeClass
+    public static void startHttp() throws Exception {
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+
+        BiConsumer<String, String> registerContext = (path, v) ->{
+            final byte[] message = v.getBytes(UTF_8);
+            httpServer.createContext(path, (s) -> {
+                s.sendResponseHeaders(RestStatus.OK.getStatus(), message.length);
+                OutputStream responseBody = s.getResponseBody();
+                responseBody.write(message);
+                responseBody.close();
+            });
+        };
+        registerContext.accept("/latest/meta-data/local-ipv4","127.0.0.1");
+        registerContext.accept("/latest/meta-data/public-ipv4","165.168.10.2");
+        registerContext.accept("/latest/meta-data/public-hostname","165.168.10.3");
+        registerContext.accept("/latest/meta-data/local-hostname","10.10.10.5");
+
+        httpServer.start();
+    }
+
+    @Before
+    public void setup() {
+        // redirect EC2 metadata service to httpServer
+        AccessController.doPrivileged((PrivilegedAction<String>) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY,
+            "http://" + httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort()));
+    }
+
+    @AfterClass
+    public static void stopHttp() {
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
     /**
      * Test for network.host: _ec2_
      */
     public void testNetworkHostEc2() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2_")
-                .build();
+        resolveEc2("_ec2_", InetAddress.getByName("127.0.0.1"));
+    }
+
+    /**
+     * Test for network.host: _ec2_
+     */
+    public void testNetworkHostUnableToResolveEc2() {
+        // redirect EC2 metadata service to unknown location
+        AccessController.doPrivileged((PrivilegedAction<String>) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY,
+            "http://127.0.0.1/"));
 
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
         try {
-            networkService.resolveBindHostAddresses(null);
+            resolveEc2("_ec2_", (InetAddress[]) null);
         } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
+            assertThat(e.getMessage(),
+                equalTo("IOException caught when fetching InetAddress from [http://127.0.0.1//latest/meta-data/local-ipv4]"));
         }
     }
 
@@ -55,102 +118,58 @@ public void testNetworkHostEc2() throws IOException {
      * Test for network.host: _ec2:publicIp_
      */
     public void testNetworkHostEc2PublicIp() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicIp_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-ipv4"));
-        }
+        resolveEc2("_ec2:publicIp_", InetAddress.getByName("165.168.10.2"));
     }
 
     /**
      * Test for network.host: _ec2:privateIp_
      */
     public void testNetworkHostEc2PrivateIp() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateIp_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
-        }
+        resolveEc2("_ec2:privateIp_", InetAddress.getByName("127.0.0.1"));
     }
 
     /**
      * Test for network.host: _ec2:privateIpv4_
      */
     public void testNetworkHostEc2PrivateIpv4() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateIpv4_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-ipv4"));
-        }
+        resolveEc2("_ec2:privateIpv4_", InetAddress.getByName("127.0.0.1"));
     }
 
     /**
      * Test for network.host: _ec2:privateDns_
      */
     public void testNetworkHostEc2PrivateDns() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:privateDns_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("local-hostname"));
-        }
+        resolveEc2("_ec2:privateDns_", InetAddress.getByName("10.10.10.5"));
     }
 
     /**
      * Test for network.host: _ec2:publicIpv4_
      */
     public void testNetworkHostEc2PublicIpv4() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicIpv4_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-ipv4"));
-        }
+        resolveEc2("_ec2:publicIpv4_", InetAddress.getByName("165.168.10.2"));
     }
 
     /**
      * Test for network.host: _ec2:publicDns_
      */
     public void testNetworkHostEc2PublicDns() throws IOException {
+        resolveEc2("_ec2:publicDns_", InetAddress.getByName("165.168.10.3"));
+    }
+
+    private InetAddress[] resolveEc2(String host, InetAddress ... expected) throws IOException {
         Settings nodeSettings = Settings.builder()
-                .put("network.host", "_ec2:publicDns_")
-                .build();
+            .put("network.host", host)
+            .build();
 
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
-        // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach.
-        try {
-            networkService.resolveBindHostAddresses(null);
-        } catch (IOException e) {
-            assertThat(e.getMessage(), containsString("public-hostname"));
+        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
+
+        InetAddress[] addresses = networkService.resolveBindHostAddresses(
+            NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING.get(nodeSettings).toArray(Strings.EMPTY_ARRAY));
+        if (expected == null) {
+            fail("We should get an IOException, resolved addressed:" + Arrays.toString(addresses));
         }
+        assertThat(addresses, arrayContaining(expected));
+        return addresses;
     }
 
     /**
@@ -158,11 +177,7 @@ public void testNetworkHostEc2PublicDns() throws IOException {
      * network.host: _local_
      */
     public void testNetworkHostCoreLocal() throws IOException {
-        Settings nodeSettings = Settings.builder()
-                .put("network.host", "_local_")
-                .build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver(nodeSettings)));
+        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
         InetAddress[] addresses = networkService.resolveBindHostAddresses(null);
         assertThat(addresses, arrayContaining(networkService.resolveBindHostAddresses(new String[] { "_local_" })));
     }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
index aab6e0c74ecdb..116bf1842d065 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
@@ -94,6 +94,7 @@ public Collection<Instance> instances() {
         return instances;
     }
 
+    private final Settings settings;
     private Compute client;
     private TimeValue refreshInterval = null;
     private long lastRefresh;
@@ -107,7 +108,7 @@ public Collection<Instance> instances() {
     private final boolean validateCerts;
 
     public GceInstancesServiceImpl(Settings settings) {
-        super(settings);
+        this.settings = settings;
         this.validateCerts = GCE_VALIDATE_CERTIFICATES.get(settings);
         this.project = resolveProject();
         this.zones = resolveZones();
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
index c736862d426de..ca25fde742907 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
@@ -44,11 +44,14 @@ public class GceMetadataService extends AbstractLifecycleComponent {
     public static final Setting<String> GCE_HOST =
         new Setting<>("cloud.gce.host", "http://metadata.google.internal", Function.identity(), Setting.Property.NodeScope);
 
+    private final Settings settings;
+
     /** Global instance of the HTTP transport. */
     private HttpTransport gceHttpTransport;
 
     public GceMetadataService(Settings settings) {
         super(settings);
+        this.settings = settings;
     }
 
     protected synchronized HttpTransport getGceHttpTransport() throws GeneralSecurityException, IOException {
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java
index 71e9fbc7804df..064fe606244ee 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.cloud.gce;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 
 public class GceModule extends AbstractModule {
@@ -29,7 +29,7 @@ public class GceModule extends AbstractModule {
     static Class<? extends GceInstancesService> computeServiceImpl = GceInstancesServiceImpl.class;
 
     protected final Settings settings;
-    protected final Logger logger = Loggers.getLogger(GceModule.class);
+    protected final Logger logger = LogManager.getLogger(GceModule.class);
 
     public GceModule(Settings settings) {
         this.settings = settings;
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
index 46c4ac7bac547..e53a1e241bb1d 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
@@ -22,9 +22,7 @@
 import org.elasticsearch.cloud.gce.GceMetadataService;
 import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
-import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -39,7 +37,7 @@
  * <li>_gce:hostname_</li>
  * </ul>
  */
-public class GceNameResolver extends AbstractComponent implements CustomNameResolver {
+public class GceNameResolver implements CustomNameResolver {
 
     private final GceMetadataService gceMetadataService;
 
@@ -73,8 +71,7 @@ private enum GceAddressResolverType {
     /**
      * Construct a {@link CustomNameResolver}.
      */
-    public GceNameResolver(Settings settings, GceMetadataService gceMetadataService) {
-        super(settings);
+    public GceNameResolver(GceMetadataService gceMetadataService) {
         this.gceMetadataService = gceMetadataService;
     }
 
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
index 36f8aa36b34d0..2d1bb07b23909 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
@@ -58,6 +58,7 @@ static final class Status {
         private static final String TERMINATED = "TERMINATED";
     }
 
+    private final Settings settings;
     private final GceInstancesService gceInstancesService;
     private TransportService transportService;
     private NetworkService networkService;
@@ -73,7 +74,7 @@ static final class Status {
     public GceUnicastHostsProvider(Settings settings, GceInstancesService gceInstancesService,
             TransportService transportService,
             NetworkService networkService) {
-        super(settings);
+        this.settings = settings;
         this.gceInstancesService = gceInstancesService;
         this.transportService = transportService;
         this.networkService = networkService;
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index d95886e349463..af318af312753 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -21,19 +21,19 @@
 
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.util.ClassInfo;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.Booleans;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.cloud.gce.GceInstancesService;
 import org.elasticsearch.cloud.gce.GceInstancesServiceImpl;
 import org.elasticsearch.cloud.gce.GceMetadataService;
 import org.elasticsearch.cloud.gce.network.GceNameResolver;
 import org.elasticsearch.cloud.gce.util.Access;
-import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.discovery.gce.GceUnicastHostsProvider;
 import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.plugins.DiscoveryPlugin;
@@ -57,7 +57,7 @@ public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
 
     public static final String GCE = "gce";
     protected final Settings settings;
-    private static final Logger logger = Loggers.getLogger(GceDiscoveryPlugin.class);
+    private static final Logger logger = LogManager.getLogger(GceDiscoveryPlugin.class);
 
     // stashed when created in order to properly close
     private final SetOnce<GceInstancesService> gceInstancesService = new SetOnce<>();
@@ -96,7 +96,7 @@ public Map<String, Supplier<UnicastHostsProvider>> getZenHostsProviders(Transpor
     @Override
     public NetworkService.CustomNameResolver getCustomNameResolver(Settings settings) {
         logger.debug("Register _gce_, _gce:xxx network names");
-        return new GceNameResolver(settings, new GceMetadataService(settings));
+        return new GceNameResolver(new GceMetadataService(settings));
     }
 
     @Override
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
index 3a34e3629db80..03525eb266ba7 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceMockUtils.java
@@ -26,18 +26,18 @@
 import com.google.api.client.testing.http.MockHttpTransport;
 import com.google.api.client.testing.http.MockLowLevelHttpRequest;
 import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 
 public class GceMockUtils {
-    protected static final Logger logger = Loggers.getLogger(GceMockUtils.class);
+    protected static final Logger logger = LogManager.getLogger(GceMockUtils.class);
 
     public static final String GCE_METADATA_URL = "http://metadata.google.internal/computeMetadata/v1/";
 
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java
index 1fe1297904bda..94f2959917d5b 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java
@@ -107,7 +107,7 @@ private void resolveGce(String gceNetworkSetting, InetAddress[] expected) throws
                 .build();
 
         GceMetadataServiceMock mock = new GceMetadataServiceMock(nodeSettings);
-        NetworkService networkService = new NetworkService(Collections.singletonList(new GceNameResolver(nodeSettings, mock)));
+        NetworkService networkService = new NetworkService(Collections.singletonList(new GceNameResolver(mock)));
         try {
             InetAddress[] addresses = networkService.resolveBindHostAddresses(
                 NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING.get(nodeSettings).toArray(Strings.EMPTY_ARRAY));
diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml
index bbb0b44ef1d45..b864edaa2a962 100644
--- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml
+++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/20_whitelist.yml
@@ -11,8 +11,8 @@
         indices.refresh: {}
 
     - do:
-        index: test
         search:
+            index: test
             body:
                 query:
                     match_all: {}
diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml
index b659263729607..1dbaf655bf501 100644
--- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml
+++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/30_static.yml
@@ -11,8 +11,8 @@
     indices.refresh: {}
 
 - do:
-    index: test
     search:
+      index: test
       body:
         query:
           match_all: {}
diff --git a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml
index 6cb7e4f3d401d..87afe9602dc35 100644
--- a/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml
+++ b/plugins/examples/painless-whitelist/src/test/resources/rest-api-spec/test/painless_whitelist/40_instance.yml
@@ -11,8 +11,8 @@
     indices.refresh: {}
 
 - do:
-    index: test
     search:
+      index: test
       body:
         query:
           match_all: {}
@@ -26,8 +26,8 @@
 - match: { hits.hits.0.fields.sNum1.0: 2 }
 
 - do:
-    index: test
     search:
+      index: test
       body:
         query:
           match_all: {}
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
index b420e8d0a1198..a185e038582e0 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
@@ -19,11 +19,11 @@
 package org.elasticsearch.ingest.geoip;
 
 import com.maxmind.geoip2.DatabaseReader;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.CheckedSupplier;
-import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.core.internal.io.IOUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -34,7 +34,7 @@
  */
 final class DatabaseReaderLazyLoader implements Closeable {
 
-    private static final Logger LOGGER = Loggers.getLogger(DatabaseReaderLazyLoader.class);
+    private static final Logger LOGGER = LogManager.getLogger(DatabaseReaderLazyLoader.class);
 
     private final String databaseFileName;
     private final CheckedSupplier<DatabaseReader, IOException> loader;
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index e710a5971173e..1eae00ea49087 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -63,7 +63,7 @@ public void testSizeEnabled() throws Exception {
         boolean points = false;
         for (IndexableField field : doc.rootDoc().getFields("_size")) {
             stored |= field.fieldType().stored();
-            points |= field.fieldType().pointDimensionCount() > 0;
+            points |= field.fieldType().pointIndexDimensionCount() > 0;
         }
         assertTrue(stored);
         assertTrue(points);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index 08041d54cd14a..e7c8478c3ed39 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -21,12 +21,12 @@
 
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
-import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -37,7 +37,7 @@
 
 public class AzureBlobContainer extends AbstractBlobContainer {
 
-    private final Logger logger = Loggers.getLogger(AzureBlobContainer.class);
+    private final Logger logger = LogManager.getLogger(AzureBlobContainer.class);
     private final AzureBlobStore blobStore;
 
     private final String keyPath;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index d519af370eb9e..a6eb849aa17ac 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
@@ -49,9 +48,8 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
     private final String container;
     private final LocationMode locationMode;
 
-    public AzureBlobStore(RepositoryMetaData metadata, Settings settings, AzureStorageService service)
+    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service)
             throws URISyntaxException, StorageException {
-        super(settings);
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
         this.clientName = Repository.CLIENT_NAME.get(metadata.settings());
         this.service = service;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index d5542444ac7de..f4b7b132c9bc6 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -71,7 +71,6 @@ public class AzureStorageService extends AbstractComponent {
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
 
     public AzureStorageService(Settings settings) {
-        super(settings);
         // eagerly load client settings so that secure settings are read
         final Map<String, AzureStorageSettings> clientsSettings = AzureStorageSettings.load(settings);
         refreshAndClearCache(clientsSettings);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index f1b3066432a88..1649e4dead8d7 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -58,7 +58,7 @@ public final class AzureStorageSettings {
         key -> SecureSetting.secureString(key, null));
 
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
-    private static final Setting<Integer> MAX_RETRIES_SETTING =
+    public static final Setting<Integer> MAX_RETRIES_SETTING =
         Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
             (key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope),
             ACCOUNT_SETTING, KEY_SETTING);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index d2eebcff75450..05f07cc374c33 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -64,6 +64,7 @@ public List<Setting<?>> getSettings() {
             AzureStorageSettings.KEY_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
+            AzureStorageSettings.MAX_RETRIES_SETTING,
             AzureStorageSettings.PROXY_TYPE_SETTING,
             AzureStorageSettings.PROXY_HOST_SETTING,
             AzureStorageSettings.PROXY_PORT_SETTING
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 4182e8f1f752b..90f536385d523 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -127,7 +127,7 @@ protected BlobStore getBlobStore() {
      */
     @Override
     protected AzureBlobStore createBlobStore() throws URISyntaxException, StorageException {
-        final AzureBlobStore blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
+        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService);
 
         logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
             "using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index a92c13e4f85a2..f7860356693cc 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -26,10 +26,12 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
@@ -37,6 +39,7 @@
 import java.net.URISyntaxException;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.cloud.azure.storage.AzureStorageService.blobNameFromUri;
@@ -79,10 +82,24 @@ public void testReadSecuredSettings() {
         assertThat(loadedSettings.get("azure3").getEndpointSuffix(), equalTo("my_endpoint_suffix"));
     }
 
+    private AzureRepositoryPlugin pluginWithSettingsValidation(Settings settings) {
+        final AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings);
+        new SettingsModule(settings, plugin.getSettings(), Collections.emptyList(), Collections.emptySet());
+        return plugin;
+    }
+
+    private AzureStorageService storageServiceWithSettingsValidation(Settings settings) {
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            return plugin.azureStoreService;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     public void testCreateClientWithEndpointSuffix() throws IOException {
         final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix").build();
-        try (AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings)) {
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client1 = azureStorageService.client("azure1").v1();
             assertThat(client1.getEndpoint().toString(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
@@ -104,7 +121,7 @@ public void testReinitClientSettings() throws IOException {
         secureSettings2.setString("azure.client.azure3.account", "myaccount23");
         secureSettings2.setString("azure.client.azure3.key", encodeKey("mykey23"));
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
-        try (AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings1)) {
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount11.blob.core.windows.net"));
@@ -136,7 +153,7 @@ public void testReinitClientEmptySettings() throws IOException {
         secureSettings.setString("azure.client.azure1.account", "myaccount1");
         secureSettings.setString("azure.client.azure1.key", encodeKey("mykey11"));
         final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
-        try (AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings)) {
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
@@ -160,7 +177,7 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings2.setString("azure.client.azure1.account", "myaccount1");
         // missing key
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
-        try (AzureRepositoryPlugin plugin = new AzureRepositoryPlugin(settings1)) {
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
@@ -173,7 +190,7 @@ public void testReinitClientWrongSettings() throws IOException {
     }
 
     public void testGetSelectedClientNonExisting() {
-        final AzureStorageService azureStorageService = new AzureStorageService(buildSettings());
+        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
         final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4"));
         assertThat(e.getMessage(), is("Unable to find client with name [azure4]"));
     }
@@ -199,7 +216,7 @@ public void testGetSelectedClientDefaultTimeout() {
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.timeout", "30s")
             .build();
-        final AzureStorageService azureStorageService = new AzureStorageService(timeoutSettings);
+        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
         final CloudBlobClient client1 = azureStorageService.client("azure1").v1();
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), nullValue());
         final CloudBlobClient client3 = azureStorageService.client("azure3").v1();
@@ -207,13 +224,13 @@ public void testGetSelectedClientDefaultTimeout() {
     }
 
     public void testGetSelectedClientNoTimeout() {
-        final AzureStorageService azureStorageService = new AzureStorageService(buildSettings());
+        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
         final CloudBlobClient client1 = azureStorageService.client("azure1").v1();
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(nullValue()));
     }
 
     public void testGetSelectedClientBackoffPolicy() {
-        final AzureStorageService azureStorageService = new AzureStorageService(buildSettings());
+        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
         final CloudBlobClient client1 = azureStorageService.client("azure1").v1();
         assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
         assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
@@ -225,7 +242,7 @@ public void testGetSelectedClientBackoffPolicyNbRetries() {
             .put("azure.client.azure1.max_retries", 7)
             .build();
 
-        final AzureStorageService azureStorageService = new AzureStorageService(timeoutSettings);
+        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
         final CloudBlobClient client1 = azureStorageService.client("azure1").v1();
         assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), is(notNullValue()));
         assertThat(client1.getDefaultRequestOptions().getRetryPolicyFactory(), instanceOf(RetryExponentialRetry.class));
@@ -235,7 +252,7 @@ public void testNoProxy() {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .build();
-        final AzureStorageService mock = new AzureStorageService(settings);
+        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
         assertThat(mock.storageSettings.get("azure1").getProxy(), nullValue());
         assertThat(mock.storageSettings.get("azure2").getProxy(), nullValue());
         assertThat(mock.storageSettings.get("azure3").getProxy(), nullValue());
@@ -248,7 +265,7 @@ public void testProxyHttp() throws UnknownHostException {
             .put("azure.client.azure1.proxy.port", 8080)
             .put("azure.client.azure1.proxy.type", "http")
             .build();
-        final AzureStorageService mock = new AzureStorageService(settings);
+        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
         final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy();
 
         assertThat(azure1Proxy, notNullValue());
@@ -268,7 +285,7 @@ public void testMultipleProxies() throws UnknownHostException {
             .put("azure.client.azure2.proxy.port", 8081)
             .put("azure.client.azure2.proxy.type", "http")
             .build();
-        final AzureStorageService mock = new AzureStorageService(settings);
+        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
         final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy();
         assertThat(azure1Proxy, notNullValue());
         assertThat(azure1Proxy.type(), is(Proxy.Type.HTTP));
@@ -287,7 +304,7 @@ public void testProxySocks() throws UnknownHostException {
             .put("azure.client.azure1.proxy.port", 8080)
             .put("azure.client.azure1.proxy.type", "socks")
             .build();
-        final AzureStorageService mock = new AzureStorageService(settings);
+        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
         final Proxy azure1Proxy = mock.storageSettings.get("azure1").getProxy();
         assertThat(azure1Proxy, notNullValue());
         assertThat(azure1Proxy.type(), is(Proxy.Type.SOCKS));
@@ -302,7 +319,7 @@ public void testProxyNoHost() {
             .put("azure.client.azure1.proxy.port", 8080)
             .put("azure.client.azure1.proxy.type", randomFrom("socks", "http"))
             .build();
-        final SettingsException e = expectThrows(SettingsException.class, () -> new AzureStorageService(settings));
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
         assertEquals("Azure Proxy type has been set but proxy host or port is not defined.", e.getMessage());
     }
 
@@ -313,7 +330,7 @@ public void testProxyNoPort() {
             .put("azure.client.azure1.proxy.type", randomFrom("socks", "http"))
             .build();
 
-        final SettingsException e = expectThrows(SettingsException.class, () -> new AzureStorageService(settings));
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
         assertEquals("Azure Proxy type has been set but proxy host or port is not defined.", e.getMessage());
     }
 
@@ -324,7 +341,7 @@ public void testProxyNoType() {
             .put("azure.client.azure1.proxy.port", 8080)
             .build();
 
-        final SettingsException e = expectThrows(SettingsException.class, () -> new AzureStorageService(settings));
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
         assertEquals("Azure Proxy port or host have been set but proxy type is not defined.", e.getMessage());
     }
 
@@ -336,7 +353,7 @@ public void testProxyWrongHost() {
             .put("azure.client.azure1.proxy.port", 8080)
             .build();
 
-        final SettingsException e = expectThrows(SettingsException.class, () -> new AzureStorageService(settings));
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
         assertEquals("Azure proxy host is unknown.", e.getMessage());
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
index 85ca44205aa94..7248254b7ca7b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
@@ -36,7 +36,7 @@ protected BlobStore newBlobStore() throws IOException {
         try {
             RepositoryMetaData repositoryMetaData = new RepositoryMetaData("azure", "ittest", Settings.EMPTY);
             AzureStorageServiceMock client = new AzureStorageServiceMock();
-            return new AzureBlobStore(repositoryMetaData, Settings.EMPTY, client);
+            return new AzureBlobStore(repositoryMetaData, client);
         } catch (URISyntaxException | StorageException e) {
             throw new IOException(e);
         }
diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
index c7b8949a11335..bb8f148fc8ab1 100644
--- a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
+++ b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
@@ -11,7 +11,7 @@ setup:
           settings:
             bucket: ${bucket}
             client: "integration_test"
-            base_path: ${base_path}
+            base_path: "${base_path}"
 
 ---
 "Snapshot/Restore with repository-gcs":
@@ -23,7 +23,7 @@ setup:
 
   - match: { repository.settings.bucket : ${bucket} }
   - match: { repository.settings.client : "integration_test" }
-  - match: { repository.settings.base_path : ${base_path} }
+  - match: { repository.settings.base_path : "${base_path}" }
 
   # Index documents
   - do:
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index 1e94467f5a57e..7894f9fc7df63 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.internal.io.Streams;
 
 import java.io.ByteArrayOutputStream;
@@ -69,8 +68,7 @@ class GoogleCloudStorageBlobStore extends AbstractComponent implements BlobStore
     private final String clientName;
     private final GoogleCloudStorageService storageService;
 
-    GoogleCloudStorageBlobStore(Settings settings, String bucketName, String clientName, GoogleCloudStorageService storageService) {
-        super(settings);
+    GoogleCloudStorageBlobStore(String bucketName, String clientName, GoogleCloudStorageService storageService) {
         this.bucketName = bucketName;
         this.clientName = clientName;
         this.storageService = storageService;
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
index 12e7fd26ff565..3186d2547a327 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java
@@ -38,14 +38,14 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin
     final GoogleCloudStorageService storageService;
 
     public GoogleCloudStoragePlugin(final Settings settings) {
-        this.storageService = createStorageService(settings);
+        this.storageService = createStorageService();
         // eagerly load client settings so that secure settings are readable (not closed)
         reload(settings);
     }
 
     // overridable for tests
-    protected GoogleCloudStorageService createStorageService(Settings settings) {
-        return new GoogleCloudStorageService(settings);
+    protected GoogleCloudStorageService createStorageService() {
+        return new GoogleCloudStorageService();
     }
 
     @Override
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index f3a93c8647cf4..379fc10bc86f0 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -78,6 +79,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
     static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =
             timeSetting("http.connect_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
 
+    private final Settings settings;
     private final GoogleCloudStorageService storageService;
     private final BlobPath basePath;
     private final boolean compress;
@@ -89,6 +91,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
                                         NamedXContentRegistry namedXContentRegistry,
                                         GoogleCloudStorageService storageService) {
         super(metadata, environment.settings(), namedXContentRegistry);
+        this.settings = environment.settings();
         this.storageService = storageService;
 
         String basePath = BASE_PATH.get(metadata.settings());
@@ -132,7 +135,7 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
 
     @Override
     protected GoogleCloudStorageBlobStore createBlobStore() {
-        return new GoogleCloudStorageBlobStore(settings, bucket, clientName, storageService);
+        return new GoogleCloudStorageBlobStore(bucket, clientName, storageService);
     }
 
     @Override
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index 55c9573ce9cd7..37b40cb2cbf79 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.LazyInitializable;
 
@@ -70,10 +69,6 @@ public class GoogleCloudStorageService extends AbstractComponent {
     @Deprecated
     private TimeValue readTimeout;
 
-    public GoogleCloudStorageService(final Settings settings) {
-        super(settings);
-    }
-
     /**
      * Refreshes the client settings and clears the client cache. Subsequent calls to
      * {@code GoogleCloudStorageService#client} will return new clients constructed
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
index 0cc1243f28311..2f23011d4d9b7 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
 
 import java.util.Locale;
@@ -42,6 +41,6 @@ protected BlobStore newBlobStore() {
         } catch (final Exception e) {
             throw new RuntimeException(e);
         }
-        return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucketName, clientName, storageService);
+        return new GoogleCloudStorageBlobStore(bucketName, clientName, storageService);
     }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index 6d5c1bbf85310..db166a228b576 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -79,17 +79,12 @@ public MockGoogleCloudStoragePlugin(final Settings settings) {
         }
 
         @Override
-        protected GoogleCloudStorageService createStorageService(Settings settings) {
-            return new MockGoogleCloudStorageService(settings);
+        protected GoogleCloudStorageService createStorageService() {
+            return new MockGoogleCloudStorageService();
         }
     }
 
     public static class MockGoogleCloudStorageService extends GoogleCloudStorageService {
-
-        MockGoogleCloudStorageService(Settings settings) {
-            super(settings);
-        }
-
         @Override
         public Storage client(String clientName) {
             return new MockStorage(BUCKET, blobs);
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
index 4634bd3274a70..e2adfed94bbc9 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.repositories.gcs;
 
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreTestCase;
 
 import java.util.Locale;
@@ -42,6 +41,6 @@ protected BlobStore newBlobStore() {
         } catch (final Exception e) {
             throw new RuntimeException(e);
         }
-        return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucketName, clientName, storageService);
+        return new GoogleCloudStorageBlobStore(bucketName, clientName, storageService);
     }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryDeprecationTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryDeprecationTests.java
index 0e11286181e9a..c45947bc7feb5 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryDeprecationTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryDeprecationTests.java
@@ -43,7 +43,7 @@ public void testDeprecatedSettings() throws Exception {
         final RepositoryMetaData repositoryMetaData = new RepositoryMetaData("test", "gcs", repositorySettings);
         final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         new GoogleCloudStorageRepository(repositoryMetaData, environment, NamedXContentRegistry.EMPTY,
-                new GoogleCloudStorageService(Settings.EMPTY) {
+                new GoogleCloudStorageService() {
                     @Override
                     public Storage client(String clientName) throws IOException {
                         return new MockStorage("test", new ConcurrentHashMap<>());
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
index 5fa2f3c83a2f2..e99310645999e 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java
@@ -63,7 +63,7 @@ public void testClientInitializer() throws Exception {
                 .put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
                 .put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
                 .build();
-        final GoogleCloudStorageService service = new GoogleCloudStorageService(settings);
+        final GoogleCloudStorageService service = new GoogleCloudStorageService();
         service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
         final String deprecatedApplicationName = randomBoolean() ? null : "deprecated_" + randomAlphaOfLength(4);
         service.setOverrideApplicationName(deprecatedApplicationName);
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index 75ac06dcfaa93..55641041a7051 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -18,15 +18,6 @@
  */
 package org.elasticsearch.repositories.hdfs;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.UnknownHostException;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Locale;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.FileContext;
@@ -36,13 +27,13 @@
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -50,9 +41,18 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Locale;
+
 public final class HdfsRepository extends BlobStoreRepository {
 
-    private static final Logger LOGGER = Loggers.getLogger(HdfsRepository.class);
+    private static final Logger LOGGER = LogManager.getLogger(HdfsRepository.class);
 
     private static final String CONF_SECURITY_PRINCIPAL = "security.principal";
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
index 7715c7086a67b..e1ffc7a22d44c 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.BlobStoreException;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
@@ -55,9 +54,8 @@ class S3BlobStore extends AbstractComponent implements BlobStore {
 
     private final StorageClass storageClass;
 
-    S3BlobStore(Settings settings, S3Service service, String clientName, String bucket, boolean serverSideEncryption,
+    S3BlobStore(S3Service service, String clientName, String bucket, boolean serverSideEncryption,
                 ByteSizeValue bufferSize, String cannedACL, String storageClass) {
-        super(settings);
         this.service = service;
         this.clientName = clientName;
         this.bucket = bucket;
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index a2f600a34ae2c..e0e34e40f3cf8 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -148,6 +148,8 @@ class S3Repository extends BlobStoreRepository {
      */
     static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path");
 
+    private final Settings settings;
+
     private final S3Service service;
 
     private final String bucket;
@@ -178,6 +180,7 @@ class S3Repository extends BlobStoreRepository {
                  final NamedXContentRegistry namedXContentRegistry,
                  final S3Service service) {
         super(metadata, settings, namedXContentRegistry);
+        this.settings = settings;
         this.service = service;
 
         // Parse and validate the user's S3 Storage Class setting
@@ -242,7 +245,7 @@ class S3Repository extends BlobStoreRepository {
     protected S3BlobStore createBlobStore() {
         if (reference != null) {
             assert S3ClientSettings.checkDeprecatedCredentials(metadata.settings()) : metadata.name();
-            return new S3BlobStore(settings, service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass) {
+            return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass) {
                 @Override
                 public AmazonS3Reference clientReference() {
                     if (reference.tryIncRef()) {
@@ -253,7 +256,7 @@ public AmazonS3Reference clientReference() {
                 }
             };
         } else {
-            return new S3BlobStore(settings, service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
+            return new S3BlobStore(service, clientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
         }
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
index da3219f2aef08..a2f9da5f846ef 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
@@ -64,7 +64,7 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     protected final S3Service service;
 
     public S3RepositoryPlugin(final Settings settings) {
-        this(settings, new S3Service(settings));
+        this(settings, new S3Service());
     }
 
     S3RepositoryPlugin(final Settings settings, final S3Service service) {
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
index 7fd5659c8e1c4..e4d8abf65f836 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java
@@ -46,10 +46,6 @@ class S3Service extends AbstractComponent implements Closeable {
     private volatile Map<String, AmazonS3Reference> clientsCache = emptyMap();
     private volatile Map<String, S3ClientSettings> clientsSettings = emptyMap();
 
-    S3Service(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Refreshes the settings for the AmazonS3 clients and clears the cache of
      * existing clients. New clients will be build using these new settings. Old
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
index 1c3c47943a06e..ec5d5578a03a2 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java
@@ -68,11 +68,6 @@ public HeadBucketResult headBucket(HeadBucketRequest headBucketRequest) throws A
         }
 
         static final class ProxyS3Service extends S3Service {
-
-            ProxyS3Service(Settings settings) {
-                super(settings);
-            }
-
             @Override
             AmazonS3 buildClient(final S3ClientSettings clientSettings) {
                 final AmazonS3 client = super.buildClient(clientSettings);
@@ -82,7 +77,7 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) {
         }
 
         ProxyS3RepositoryPlugin(Settings settings) {
-            super(settings, new ProxyS3Service(settings));
+            super(settings, new ProxyS3Service());
         }
 
         @Override
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
index bdb5f8468cee2..49c76e289d243 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -122,7 +122,7 @@ public TestS3RepositoryPlugin(final Settings settings) {
         @Override
         public Map<String, Repository.Factory> getRepositories(final Environment env, final NamedXContentRegistry registry) {
             return Collections.singletonMap(S3Repository.TYPE,
-                    (metadata) -> new S3Repository(metadata, env.settings(), registry, new S3Service(env.settings()) {
+                    (metadata) -> new S3Repository(metadata, env.settings(), registry, new S3Service() {
                         @Override
                         AmazonS3 buildClient(S3ClientSettings clientSettings) {
                             return new MockAmazonS3(blobs, bucket, serverSideEncryption, cannedACL, storageClass);
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
index 55df03ff34a3f..a44ad706b2361 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java
@@ -24,7 +24,6 @@
 import com.amazonaws.services.s3.model.StorageClass;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.BlobStoreException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.repositories.ESBlobStoreTestCase;
@@ -117,13 +116,13 @@ public static S3BlobStore randomMockS3BlobStore() {
 
         final String theClientName = randomAlphaOfLength(4);
         final AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass);
-        final S3Service service = new S3Service(Settings.EMPTY) {
+        final S3Service service = new S3Service() {
             @Override
             public synchronized AmazonS3Reference client(String clientName) {
                 assert theClientName.equals(clientName);
                 return new AmazonS3Reference(client);
             }
         };
-        return new S3BlobStore(Settings.EMPTY, service, theClientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
+        return new S3BlobStore(service, theClientName, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass);
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index b76af23402c05..ecfa8e8d97dc1 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -57,10 +57,6 @@ public void shutdown() {
     }
 
     private static class DummyS3Service extends S3Service {
-        DummyS3Service() {
-            super(Settings.EMPTY);
-        }
-
         @Override
         public AmazonS3Reference client(String clientName) {
             return new AmazonS3Reference(new DummyS3Client());
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
index 39ce992b7a58e..66f38ca043776 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
@@ -12,7 +12,7 @@ setup:
           settings:
             bucket: ${permanent_bucket}
             client: integration_test_permanent
-            base_path: ${permanent_base_path}
+            base_path: "${permanent_base_path}"
             canned_acl: private
             storage_class: standard
 
@@ -26,7 +26,7 @@ setup:
 
   - match: { repository_permanent.settings.bucket : ${permanent_bucket} }
   - match: { repository_permanent.settings.client : "integration_test_permanent" }
-  - match: { repository_permanent.settings.base_path : ${permanent_base_path} }
+  - match: { repository_permanent.settings.base_path : "${permanent_base_path}" }
   - match: { repository_permanent.settings.canned_acl : "private" }
   - match: { repository_permanent.settings.storage_class : "standard" }
   - is_false: repository_permanent.settings.access_key
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
index 497d85db752db..3f5685aa561fe 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
@@ -12,7 +12,7 @@ setup:
           settings:
             bucket: ${temporary_bucket}
             client: integration_test_temporary
-            base_path: ${temporary_base_path}
+            base_path: "${temporary_base_path}"
             canned_acl: private
             storage_class: standard
 
@@ -26,7 +26,7 @@ setup:
 
   - match: { repository_temporary.settings.bucket : ${temporary_bucket} }
   - match: { repository_temporary.settings.client : "integration_test_temporary" }
-  - match: { repository_temporary.settings.base_path : ${temporary_base_path} }
+  - match: { repository_temporary.settings.base_path : "${temporary_base_path}" }
   - match: { repository_temporary.settings.canned_acl : "private" }
   - match: { repository_temporary.settings.storage_class : "standard" }
   - is_false: repository_temporary.settings.access_key
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
index 2df3b8290a19b..d021df267934d 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
@@ -12,7 +12,7 @@ setup:
           settings:
             bucket: ${ec2_bucket}
             client: integration_test_ec2
-            base_path: ${ec2_base_path}
+            base_path: "${ec2_base_path}"
             canned_acl: private
             storage_class: standard
 
@@ -26,7 +26,7 @@ setup:
 
   - match: { repository_ec2.settings.bucket : ${ec2_bucket} }
   - match: { repository_ec2.settings.client : "integration_test_ec2" }
-  - match: { repository_ec2.settings.base_path : ${ec2_base_path} }
+  - match: { repository_ec2.settings.base_path : "${ec2_base_path}" }
   - match: { repository_ec2.settings.canned_acl : "private" }
   - match: { repository_ec2.settings.storage_class : "standard" }
   - is_false: repository_ec2.settings.access_key
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
index 54929e6e3ad82..dec0476edc713 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
@@ -12,7 +12,7 @@ setup:
           settings:
             bucket: ${ecs_bucket}
             client: integration_test_ecs
-            base_path: ${ecs_base_path}
+            base_path: "${ecs_base_path}"
             canned_acl: private
             storage_class: standard
 
@@ -26,7 +26,7 @@ setup:
 
   - match: { repository_ecs.settings.bucket : ${ecs_bucket} }
   - match: { repository_ecs.settings.client : "integration_test_ecs" }
-  - match: { repository_ecs.settings.base_path : ${ecs_base_path} }
+  - match: { repository_ecs.settings.base_path : "${ecs_base_path}" }
   - match: { repository_ecs.settings.canned_acl : "private" }
   - match: { repository_ecs.settings.storage_class : "standard" }
   - is_false: repository_ecs.settings.access_key
diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle
index adb6cad288930..b1f5aff86b096 100644
--- a/qa/full-cluster-restart/build.gradle
+++ b/qa/full-cluster-restart/build.gradle
@@ -101,7 +101,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : bwcVersions.snapshotsIndexCompatible) {
+    for (final def version : bwcVersions.unreleasedIndexCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }
diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
index 576601166a035..d926317e43962 100644
--- a/qa/mixed-cluster/build.gradle
+++ b/qa/mixed-cluster/build.gradle
@@ -68,7 +68,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : bwcVersions.snapshotsWireCompatible) {
+    for (final def version : bwcVersions.unreleasedWireCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 83440ceeeb602..be8269ae941da 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -152,7 +152,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : bwcVersions.snapshotsWireCompatible) {
+    for (final def version : bwcVersions.unreleasedWireCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
index ab33d60c66b0a..91f2cabb73305 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
@@ -107,6 +107,9 @@
 
 ---
 "Find a task result record from the old cluster":
+  - skip:
+      features: headers
+
   - do:
       search:
         index: .tasks
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
index 6236a4a4d6e31..6878f3b5f34ee 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
@@ -300,7 +300,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                                                    ResourceWatcherService resourceWatcherService, ScriptService scriptService,
                                                    NamedXContentRegistry xContentRegistry, Environment environment,
                                                    NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
-            loggingFilter.set(new LoggingFilter(clusterService.getSettings(), threadPool));
+            loggingFilter.set(new LoggingFilter(threadPool));
             return Collections.emptyList();
         }
 
@@ -315,8 +315,7 @@ public static class LoggingFilter extends ActionFilter.Simple {
 
         private final ThreadPool threadPool;
 
-        public LoggingFilter(Settings settings, ThreadPool pool) {
-            super(settings);
+        public LoggingFilter(ThreadPool pool) {
             this.threadPool = pool;
         }
 
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
index ad38d89654a21..27b2f18b0919f 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
@@ -58,7 +58,7 @@ public class TestDeprecationHeaderRestAction extends BaseRestHandler {
         Setting.boolSetting("test.setting.not_deprecated", false,
                             Setting.Property.NodeScope, Setting.Property.Dynamic);
 
-    private static final Map<String, Setting<?>> SETTINGS;
+    private static final Map<String, Setting<?>> SETTINGS_MAP;
 
     static {
         Map<String, Setting<?>> settingsMap = new HashMap<>(3);
@@ -67,14 +67,17 @@ public class TestDeprecationHeaderRestAction extends BaseRestHandler {
         settingsMap.put(TEST_DEPRECATED_SETTING_TRUE2.getKey(), TEST_DEPRECATED_SETTING_TRUE2);
         settingsMap.put(TEST_NOT_DEPRECATED_SETTING.getKey(), TEST_NOT_DEPRECATED_SETTING);
 
-        SETTINGS = Collections.unmodifiableMap(settingsMap);
+        SETTINGS_MAP = Collections.unmodifiableMap(settingsMap);
     }
 
     public static final String DEPRECATED_ENDPOINT = "[/_test_cluster/deprecated_settings] exists for deprecated tests";
     public static final String DEPRECATED_USAGE = "[deprecated_settings] usage is deprecated. use [settings] instead";
 
+    private final Settings settings;
+
     public TestDeprecationHeaderRestAction(Settings settings, RestController controller) {
         super(settings);
+        this.settings = settings;
 
         controller.registerAsDeprecatedHandler(RestRequest.Method.GET, "/_test_cluster/deprecated_settings", this,
                                                DEPRECATED_ENDPOINT, deprecationLogger);
@@ -107,7 +110,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
 
             builder.startObject().startArray("settings");
             for (String setting : settings) {
-                builder.startObject().field(setting, SETTINGS.get(setting).getRaw(this.settings)).endObject();
+                builder.startObject().field(setting, SETTINGS_MAP.get(setting).getRaw(this.settings)).endObject();
             }
             builder.endArray().endObject();
             channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle
index 0e4f29fa38ad0..0878934979978 100644
--- a/qa/verify-version-constants/build.gradle
+++ b/qa/verify-version-constants/build.gradle
@@ -61,7 +61,7 @@ test.enabled = false
 
 task integTest {
   if (project.bwc_tests_enabled) {
-    final def version = bwcVersions.snapshotsIndexCompatible.first()
+    final def version = bwcVersions.unreleasedIndexCompatible.first()
     dependsOn "v${version}#bwcTest"
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 96fa4daf12b95..d14f4ab784a57 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -20,6 +20,10 @@
           "type" : "boolean",
           "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
            "type" : "boolean",
            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 28f8a07db4cc0..12e13aace4ed0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -54,6 +54,10 @@
             "type" : "boolean",
             "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
             "type" : "boolean",
             "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index a78295dd4f5a3..df1fc1b079923 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -20,6 +20,10 @@
           "type" : "boolean",
           "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
         },
+        "ignore_throttled": {
+          "type" : "boolean",
+          "description" : "Whether specified concrete, expanded or aliased indices should be ignored when throttled"
+        },
         "allow_no_indices": {
           "type" : "boolean",
           "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yml
index 05bc3fceef701..a65fb4677c8ff 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/70_source_filtering.yml
@@ -71,7 +71,7 @@
 "Deprecated _source_include and _source_exclude":
 
   - skip:
-      version: " - 6.99.99"
+      version: " - 6.5.99"
       reason: _source_include and _source_exclude are deprecated from 6.6.0
       features: "warnings"
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml
index 5c30bbc214387..c5d389e56bae3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml
@@ -66,8 +66,8 @@ setup:
 "Least impact smoke test":
 # only passing these parameters to make sure they are consumed
   - skip:
-      version: " - 6.99.99"
-      reason: "AwaitsFix'ing, see https://github.com/elastic/elasticsearch/issues/34827"
+      version: " - 6.4.99"
+      reason: "max_concurrent_shard_requests was added in 6.5.0"
   - do:
       msearch:
         max_concurrent_shard_requests: 1
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml
new file mode 100644
index 0000000000000..b0d44a8a096b0
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/270_median_absolute_deviation_metric.yml
@@ -0,0 +1,143 @@
+setup:
+  - skip:
+        version: " - 6.5.99"
+        reason:  "added in 6.6.0"
+  - do:
+        indices.create:
+            index: test
+            body:
+              settings:
+                number_of_replicas: 0
+              mappings:
+                _doc:
+                  properties:
+                    int_field:
+                      type: integer
+                    double_field:
+                      type: double
+                    incomplete_field:
+                      type: integer
+  - do:
+        bulk:
+          refresh: true
+          body:
+            - index:
+                _index: test
+                _type: _doc
+            - int_field: 100
+              double_field: 100.0
+              incomplete_field: 1000
+            - index:
+                _index: test
+                _type: _doc
+            - int_field: 200
+              double_field: 200.0
+              incomplete_field: 2000
+            - index:
+                _index: test
+                _type: _doc
+            - int_field: 300
+              double_field: 300.0
+
+---
+"basic test":
+
+  - do:
+      search:
+        body:
+          aggs:
+            mad_int:
+              median_absolute_deviation:
+                field: int_field
+            mad_double:
+              median_absolute_deviation:
+                field: double_field
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.mad_int.value: 100 }
+  - match: { aggregations.mad_double.value: 100 }
+
+---
+"with setting compression":
+
+  - do:
+      search:
+        body:
+          aggs:
+            mad_int:
+              median_absolute_deviation:
+                field: int_field
+                compression: 500
+            mad_double:
+              median_absolute_deviation:
+                field: double_field
+                compression: 500
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.mad_int.value: 100 }
+  - match: { aggregations.mad_double.value: 100 }
+
+---
+"no documents":
+
+  - do:
+      search:
+        body:
+          query:
+            bool:
+              filter:
+                term:
+                  non_existent_field: non_existent_value
+          aggs:
+            mad_no_docs:
+              median_absolute_deviation:
+                field: non_existent_field
+
+  - match: { hits.total: 0 }
+  - length: { hits.hits: 0 }
+
+  - is_false: aggregations.mad_no_docs.value
+
+---
+"missing value":
+
+  - do:
+      search:
+        body:
+          aggs:
+            mad_missing:
+              median_absolute_deviation:
+                field: incomplete_field
+                missing: 3000
+
+  - match: { hits.total: 3 }
+  - length: { hits.hits: 3 }
+
+  - match: { aggregations.mad_missing.value: 1000 }
+
+---
+"bad arguments":
+
+  - do:
+      catch: /\[compression\] must be greater than 0. Found \[0.0\] in \[mad\]/
+      search:
+        body:
+          aggs:
+            mad:
+              median_absolute_deviation:
+                field: int_field
+                compression: 0
+
+  - do:
+      catch: /\[compression\] must be greater than 0. Found \[-1.0\] in \[mad\]/
+      search:
+        body:
+          aggs:
+            mad:
+              median_absolute_deviation:
+                field: int_field
+                compression: -1
diff --git a/server/licenses/lucene-analyzers-common-7.5.0.jar.sha1 b/server/licenses/lucene-analyzers-common-7.5.0.jar.sha1
deleted file mode 100644
index 644e74b25e959..0000000000000
--- a/server/licenses/lucene-analyzers-common-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e7fbdfc2297c3ff5d194d7bef95810504e52710e
\ No newline at end of file
diff --git a/server/licenses/lucene-analyzers-common-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-analyzers-common-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..36e0c85b2b588
--- /dev/null
+++ b/server/licenses/lucene-analyzers-common-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+57a504b6795f22a25b8a1f054d9943039a85a18d
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-7.5.0.jar.sha1
deleted file mode 100644
index 40b47aea708cd..0000000000000
--- a/server/licenses/lucene-backward-codecs-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ebd10f4a1fe71a92b69c0d653136bcf9790a165
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-backward-codecs-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..146a816d44c76
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+77e50b66ce562a20b55a7580c2d5be6ee37ff23c
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.5.0.jar.sha1 b/server/licenses/lucene-core-7.5.0.jar.sha1
deleted file mode 100644
index d3e7ba985dfed..0000000000000
--- a/server/licenses/lucene-core-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-736e94305e2a0e803563c5b184877df5c7d4cb69
\ No newline at end of file
diff --git a/server/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..d8d15caa0b4c3
--- /dev/null
+++ b/server/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+ec080318251807172626c0ba8dc931276d52b15e
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.5.0.jar.sha1 b/server/licenses/lucene-grouping-7.5.0.jar.sha1
deleted file mode 100644
index e801ae199c658..0000000000000
--- a/server/licenses/lucene-grouping-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f326a63640a288c302c713fcf2965fdc827b3cca
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-grouping-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..7a8008878c2dc
--- /dev/null
+++ b/server/licenses/lucene-grouping-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+3855c79ab2ca091e84ba46d8326a149f1d67bb51
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.5.0.jar.sha1 b/server/licenses/lucene-highlighter-7.5.0.jar.sha1
deleted file mode 100644
index bd71986c8c24d..0000000000000
--- a/server/licenses/lucene-highlighter-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-59420a5e30f12885f160e49bb975ab6b5985bd3d
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-highlighter-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..46d63211696f0
--- /dev/null
+++ b/server/licenses/lucene-highlighter-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+236febdc0f3211640b265d4a6a682f98481d0e20
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.5.0.jar.sha1 b/server/licenses/lucene-join-7.5.0.jar.sha1
deleted file mode 100644
index f80a5f6fe678f..0000000000000
--- a/server/licenses/lucene-join-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-065d3c275b094383640d393579cdb7aff22bcd1d
\ No newline at end of file
diff --git a/server/licenses/lucene-join-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-join-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..19f31755df727
--- /dev/null
+++ b/server/licenses/lucene-join-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+b5ce0f6d8a6730bbcfd187401168052bd2c8450b
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.5.0.jar.sha1 b/server/licenses/lucene-memory-7.5.0.jar.sha1
deleted file mode 100644
index d535e0bae6459..0000000000000
--- a/server/licenses/lucene-memory-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b8cc0c311a823287264653fbd05f866e059d303c
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-memory-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..69f88a20d3e96
--- /dev/null
+++ b/server/licenses/lucene-memory-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+3794e1321c23187399c0b246548827a7613ea15c
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.5.0.jar.sha1 b/server/licenses/lucene-misc-7.5.0.jar.sha1
deleted file mode 100644
index a3bd848e17905..0000000000000
--- a/server/licenses/lucene-misc-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-60d0a02b9297b5423d5400a6b0aa114915232b60
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-misc-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..6f3e0b71cc3a5
--- /dev/null
+++ b/server/licenses/lucene-misc-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+605faa9cdb9ba356e289cb5be02fa687d5de1efb
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.5.0.jar.sha1 b/server/licenses/lucene-queries-7.5.0.jar.sha1
deleted file mode 100644
index 1c6c12f5b563d..0000000000000
--- a/server/licenses/lucene-queries-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bfbf786b75c60a25daf33d389efd6458e17218d9
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-queries-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..221e60a9fd14a
--- /dev/null
+++ b/server/licenses/lucene-queries-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+12ecd339212886c2b37934ef2a3af4c85d89b4e3
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.5.0.jar.sha1 b/server/licenses/lucene-queryparser-7.5.0.jar.sha1
deleted file mode 100644
index d5fb221371a2a..0000000000000
--- a/server/licenses/lucene-queryparser-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3eefb522e150f2ba0009df20f3bcfa91d60e7090
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-queryparser-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..000f3fbf36352
--- /dev/null
+++ b/server/licenses/lucene-queryparser-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+a74e9bac2cd10c878cb6aa005753267958a52101
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.5.0.jar.sha1 b/server/licenses/lucene-sandbox-7.5.0.jar.sha1
deleted file mode 100644
index 85730c40bed38..0000000000000
--- a/server/licenses/lucene-sandbox-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9fa3bb1c81179e112a62c0db6749767127c616bd
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-sandbox-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..605843f5fd9c2
--- /dev/null
+++ b/server/licenses/lucene-sandbox-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+a1bb75649b0252e4ae8f9140eb9a457b68e3f8aa
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.5.0.jar.sha1 b/server/licenses/lucene-spatial-7.5.0.jar.sha1
deleted file mode 100644
index ad600d067e3de..0000000000000
--- a/server/licenses/lucene-spatial-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8de64a6a8ad22b5849f2ab5f824917723de7a349
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-spatial-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..a11485f5d86c7
--- /dev/null
+++ b/server/licenses/lucene-spatial-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+38ef3c295d4d8b79490ff8e9451e4e65f3e6212e
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-7.5.0.jar.sha1
deleted file mode 100644
index 0c3b0dd3a6b44..0000000000000
--- a/server/licenses/lucene-spatial-extras-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b0892bbc0dc16ca07cf98897df3b3e9ed3069615
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-spatial-extras-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..76518e44c5466
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+ba81a0520b8e085a7890d91c9a90343d21bbbc65
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-7.5.0.jar.sha1
deleted file mode 100644
index 8ea67d1bde5f9..0000000000000
--- a/server/licenses/lucene-spatial3d-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-672920a7fb48624bcb84ce0ee67dd0c7bc080c94
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-spatial3d-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..4598466e048f7
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+edb2e853646d236154050fa8f1e68f908e73d10a
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.5.0.jar.sha1 b/server/licenses/lucene-suggest-7.5.0.jar.sha1
deleted file mode 100644
index 8de76642e9996..0000000000000
--- a/server/licenses/lucene-suggest-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aa7d170871711ebd4dbd367d7b8ee0f1eb5e88b7
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-7.6.0-snapshot-f9598f335b.jar.sha1 b/server/licenses/lucene-suggest-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..4013d790cfb1f
--- /dev/null
+++ b/server/licenses/lucene-suggest-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+ee715c9d8dc562b98572eb036b2aee356c23ae23
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index 3d82b51b7a633..996de8e2d6b35 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -128,6 +128,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_5_6_12 = new Version(V_5_6_12_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_5_6_13_ID = 5061399;
     public static final Version V_5_6_13 = new Version(V_5_6_13_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_14_ID = 5061499;
+    public static final Version V_5_6_14 = new Version(V_5_6_14_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_6_0_0_alpha1_ID = 6000001;
     public static final Version V_6_0_0_alpha1 = new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
     public static final int V_6_0_0_alpha2_ID = 6000002;
@@ -184,10 +186,12 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_4_2 = new Version(V_6_4_2_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_6_4_3_ID = 6040399;
     public static final Version V_6_4_3 = new Version(V_6_4_3_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+    public static final int V_6_4_4_ID = 6040499;
+    public static final Version V_6_4_4 = new Version(V_6_4_4_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_6_5_0_ID = 6050099;
     public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final int V_6_6_0_ID = 6060099;
-    public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
+    public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
 
     public static final Version CURRENT = V_6_6_0;
 
@@ -206,6 +210,8 @@ public static Version fromId(int id) {
                 return V_6_6_0;
             case V_6_5_0_ID:
                 return V_6_5_0;
+            case V_6_4_4_ID:
+                return V_6_4_4;
             case V_6_4_3_ID:
                 return V_6_4_3;
             case V_6_4_2_ID:
@@ -256,6 +262,8 @@ public static Version fromId(int id) {
                 return V_6_0_0_alpha2;
             case V_6_0_0_alpha1_ID:
                 return V_6_0_0_alpha1;
+            case V_5_6_14_ID:
+                return V_5_6_14;
             case V_5_6_13_ID:
                 return V_5_6_13;
             case V_5_6_12_ID:
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 1e25233ff51ea..ef2814c087343 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -387,7 +387,7 @@ public ActionModule(boolean transportClient, Settings settings, IndexNameExpress
         if (transportClient) {
             restController = null;
         } else {
-            restController = new RestController(settings, headers, restWrapper, nodeClient, circuitBreakerService, usageService);
+            restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java b/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
index dc42da765c6ee..d91e962b961ec 100644
--- a/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
+++ b/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -28,14 +27,13 @@
 /**
  * A generic proxy that will execute the given action against a specific node.
  */
-public class TransportActionNodeProxy<Request extends ActionRequest, Response extends ActionResponse> extends AbstractComponent {
+public class TransportActionNodeProxy<Request extends ActionRequest, Response extends ActionResponse> {
 
     private final TransportService transportService;
     private final GenericAction<Request, Response> action;
     private final TransportRequestOptions transportOptions;
 
     public TransportActionNodeProxy(Settings settings, GenericAction<Request, Response> action, TransportService transportService) {
-        super(settings);
         this.action = action;
         this.transportService = transportService;
         this.transportOptions = action.transportOptions(settings);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 9249550871c12..5bc1dfa33b28f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -67,7 +67,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
     // indices options that require every specified index to exist, expand wildcards only to open
     // indices, don't allow that no indices are resolved from wildcard expressions and resolve the
     // expressions only against indices
-    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
+    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
 
     public IndicesAliasesRequest() {
     }
@@ -472,6 +472,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (false == Strings.isEmpty(searchRouting)) {
                 builder.field(SEARCH_ROUTING.getPreferredName(), searchRouting);
             }
+            if (null != writeIndex) {
+                builder.field(IS_WRITE_INDEX.getPreferredName(), writeIndex);
+            }
             builder.endObject();
             builder.endObject();
             return builder;
@@ -491,6 +494,7 @@ public String toString() {
                     + ",routing=" + routing
                     + ",indexRouting=" + indexRouting
                     + ",searchRouting=" + searchRouting
+                    + ",writeIndex=" + writeIndex
                     + "]";
         }
 
@@ -507,12 +511,13 @@ public boolean equals(Object obj) {
                     && Objects.equals(filter, other.filter)
                     && Objects.equals(routing, other.routing)
                     && Objects.equals(indexRouting, other.indexRouting)
-                    && Objects.equals(searchRouting, other.searchRouting);
+                    && Objects.equals(searchRouting, other.searchRouting)
+                    && Objects.equals(writeIndex, other.writeIndex);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(type, indices, aliases, filter, routing, indexRouting, searchRouting);
+            return Objects.hash(type, indices, aliases, filter, routing, indexRouting, searchRouting, writeIndex);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index d29cc10e65d38..b4a4f74adb368 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -80,6 +80,7 @@
  */
 public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRequest, AnalyzeResponse> {
 
+    private final Settings settings;
     private final IndicesService indicesService;
     private final Environment environment;
 
@@ -89,6 +90,7 @@ public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterS
                                   IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) {
         super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
             AnalyzeRequest::new, ThreadPool.Names.ANALYZE);
+        this.settings = settings;
         this.indicesService = indicesService;
         this.environment = environment;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
index 249d22e7c5bd7..59300cfcecb3a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
@@ -38,7 +38,7 @@ public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest>
 
     private String[] indices;
     // Delete index should work by default on both open and closed indices.
-    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true);
+    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
 
     public DeleteIndexRequest() {
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index afbc9a554ed5e..1b385ed9d0dbc 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -71,6 +71,10 @@ public final String toString() {
         return "[" + name + ": " + value + "]";
     }
 
+    public T value() {
+        return value;
+    }
+
     /**
      * Holder for index stats used to evaluate conditions
      */
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index fe5ad65c4799b..f36636594a4d6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -196,7 +196,7 @@ public boolean isDryRun() {
         return dryRun;
     }
 
-    Map<String, Condition> getConditions() {
+    public Map<String, Condition> getConditions() {
         return conditions;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index b7e4294a5635c..356f805c24bd4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -68,8 +68,8 @@ public final class RolloverResponse extends ShardsAcknowledgedResponse implement
     RolloverResponse() {
     }
 
-    RolloverResponse(String oldIndex, String newIndex, Map<String, Boolean> conditionResults,
-                             boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) {
+    public RolloverResponse(String oldIndex, String newIndex, Map<String, Boolean> conditionResults,
+                            boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) {
         super(acknowledged, shardsAcknowledged);
         this.oldIndex = oldIndex;
         this.newIndex = newIndex;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 1734ad0514160..25b6ec0b88376 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -82,7 +82,7 @@ public TransportRolloverAction(Settings settings, TransportService transportServ
         this.createIndexService = createIndexService;
         this.indexAliasesService = indexAliasesService;
         this.client = client;
-        this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index 18c7d506c7275..96804248f62d9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -88,7 +88,7 @@ public String[] indices() {
         return indices;
     }
 
-    Settings settings() {
+    public Settings settings() {
         return settings;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
index c116bd896c81f..ffe59f1e3ae18 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
@@ -38,7 +38,7 @@ public final class ResizeResponse extends CreateIndexResponse {
     ResizeResponse() {
     }
 
-    ResizeResponse(boolean acknowledged, boolean shardsAcknowledged, String index) {
+    public ResizeResponse(boolean acknowledged, boolean shardsAcknowledged, String index) {
         super(acknowledged, shardsAcknowledged, index);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java b/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
index 81084e22377e5..00462df4d895c 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
@@ -34,7 +34,8 @@
  * semantics:
  *
  * <ul>
- *     <li><code>#hasNext()</code> determines whether the progression has more elements. Return <code>true</code> for infinite progressions</li>
+ *     <li><code>#hasNext()</code> determines whether the progression has more elements. Return <code>true</code> for infinite progressions
+ *     </li>
  *     <li><code>#next()</code> determines the next element in the progression, i.e. the next wait time period</li>
  * </ul>
  *
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
index f11444dd02288..34c18029ead5d 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java
@@ -219,12 +219,13 @@ public void close() {
      * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are flushed.
      * <p>
      * If concurrent requests are not enabled, returns {@code true} immediately.
-     * If concurrent requests are enabled, waits for up to the specified timeout for all bulk requests to complete then returns {@code true},
+     * If concurrent requests are enabled, waits for up to the specified timeout for all bulk requests to complete then returns {@code true}
      * If the specified waiting time elapses before all bulk requests complete, {@code false} is returned.
      *
      * @param timeout The maximum time to wait for the bulk requests to complete
      * @param unit    The time unit of the {@code timeout} argument
-     * @return {@code true} if all bulk requests completed and {@code false} if the waiting time elapsed before all the bulk requests completed
+     * @return {@code true} if all bulk requests completed and {@code false} if the waiting time elapsed before all the bulk requests
+     * completed
      * @throws InterruptedException If the current thread is interrupted
      */
     public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
@@ -300,7 +301,8 @@ public BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nu
      * Adds the data from the bytes to be processed by the bulk processor
      */
     public synchronized BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
-                                          @Nullable String defaultPipeline, @Nullable Object payload, XContentType xContentType) throws Exception {
+                                          @Nullable String defaultPipeline, @Nullable Object payload,
+                                          XContentType xContentType) throws Exception {
         bulkRequest.add(data, defaultIndex, defaultType, null, null, null, defaultPipeline, payload, true, xContentType);
         executeIfNeeded();
         return this;
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 9c6bcfc59ea36..a8d4e01a24924 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -376,11 +376,13 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                             } else if (PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) {
                                 pipeline = parser.text();
                             } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
-                                throw new IllegalArgumentException("Action/metadata line [" + line + "] contains a simple value for parameter [fields] while a list is expected");
+                                throw new IllegalArgumentException("Action/metadata line [" + line + "] contains a simple value for"
+                                    + " parameter [fields] while a list is expected");
                             } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                                 fetchSourceContext = FetchSourceContext.fromXContent(parser);
                             } else {
-                                throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]");
+                                throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter ["
+                                    + currentFieldName + "]");
                             }
                         } else if (token == XContentParser.Token.START_ARRAY) {
                             if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -388,21 +390,25 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                                 List<Object> values = parser.list();
                                 fields = values.toArray(new String[values.size()]);
                             } else {
-                                throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
+                                throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple "
+                                    + "value for field [" + currentFieldName + "] but found [" + token + "]");
                             }
-                        } else if (token == XContentParser.Token.START_OBJECT && SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
+                        } else if (token == XContentParser.Token.START_OBJECT && SOURCE.match(currentFieldName,
+                            parser.getDeprecationHandler())) {
                             fetchSourceContext = FetchSourceContext.fromXContent(parser);
                         } else if (token != XContentParser.Token.VALUE_NULL) {
-                            throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
+                            throw new IllegalArgumentException("Malformed action/metadata line [" + line
+                                + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                         }
                     }
                 } else if (token != XContentParser.Token.END_OBJECT) {
-                    throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected " + XContentParser.Token.START_OBJECT
-                            + " or " + XContentParser.Token.END_OBJECT + " but found [" + token + "]");
+                    throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected "
+                        + XContentParser.Token.START_OBJECT + " or " + XContentParser.Token.END_OBJECT + " but found [" + token + "]");
                 }
 
                 if ("delete".equals(action)) {
-                    add(new DeleteRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType), payload);
+                    add(new DeleteRequest(index, type, id).routing(routing).parent(parent).version(version)
+                        .versionType(versionType), payload);
                 } else {
                     nextMarker = findNextMarker(marker, from, data, length);
                     if (nextMarker == -1) {
@@ -414,21 +420,21 @@ public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Null
                     // of index request.
                     if ("index".equals(action)) {
                         if (opType == null) {
-                            internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
-                                    .setPipeline(pipeline)
+                            internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version)
+                                    .versionType(versionType).setPipeline(pipeline)
                                     .source(sliceTrimmingCarriageReturn(data, from, nextMarker,xContentType), xContentType), payload);
                         } else {
-                            internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
-                                    .create("create".equals(opType)).setPipeline(pipeline)
+                            internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version)
+                                    .versionType(versionType).create("create".equals(opType)).setPipeline(pipeline)
                                     .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), payload);
                         }
                     } else if ("create".equals(action)) {
-                        internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
-                                .create(true).setPipeline(pipeline)
+                        internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version)
+                                .versionType(versionType).create(true).setPipeline(pipeline)
                                 .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType), payload);
                     } else if ("update".equals(action)) {
-                        UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict)
-                                .version(version).versionType(versionType)
+                        UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent)
+                                .retryOnConflict(retryOnConflict).version(version).versionType(versionType)
                                 .routing(routing)
                                 .parent(parent);
                         // EMPTY is safe here because we never call namedObject
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
index 30bf2dc14773b..d474dcee3639b 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
@@ -109,7 +109,8 @@ public String buildFailureMessage() {
             BulkItemResponse response = responses[i];
             if (response.isFailed()) {
                 sb.append("\n[").append(i)
-                        .append("]: index [").append(response.getIndex()).append("], type [").append(response.getType()).append("], id [").append(response.getId())
+                        .append("]: index [").append(response.getIndex()).append("], type [")
+                        .append(response.getType()).append("], id [").append(response.getId())
                         .append("], message [").append(response.getFailureMessage()).append("]");
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 0d9daff392e2d..4ef96acf6809f 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -260,9 +260,11 @@ void createIndex(String index, TimeValue timeout, ActionListener<CreateIndexResp
         createIndexAction.execute(createIndexRequest, listener);
     }
 
-    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, DocWriteRequest request, String index, Exception e) {
+    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, DocWriteRequest request,
+                                                     String index, Exception e) {
         if (index.equals(request.index())) {
-            responses.set(idx, new BulkItemResponse(idx, request.opType(), new BulkItemResponse.Failure(request.index(), request.type(), request.id(), e)));
+            responses.set(idx, new BulkItemResponse(idx, request.opType(), new BulkItemResponse.Failure(request.index(), request.type(),
+                request.id(), e)));
             return true;
         }
         return false;
@@ -331,19 +333,23 @@ protected void doRun() throws Exception {
                             indexRequest.process(indexCreated, mappingMd, concreteIndex.getName());
                             break;
                         case UPDATE:
-                            TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest);
+                            TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(),
+                                (UpdateRequest) docWriteRequest);
                             break;
                         case DELETE:
-                            docWriteRequest.routing(metaData.resolveWriteIndexRouting(docWriteRequest.parent(), docWriteRequest.routing(), docWriteRequest.index()));
+                            docWriteRequest.routing(metaData.resolveWriteIndexRouting(docWriteRequest.parent(), docWriteRequest.routing(),
+                                docWriteRequest.index()));
                             // check if routing is required, if so, throw error if routing wasn't specified
-                            if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(), docWriteRequest.type())) {
+                            if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(),
+                                docWriteRequest.type())) {
                                 throw new RoutingMissingException(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id());
                             }
                             break;
                         default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]");
                     }
                 } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException e) {
-                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id(), e);
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(),
+                        docWriteRequest.id(), e);
                     BulkItemResponse bulkItemResponse = new BulkItemResponse(i, docWriteRequest.opType(), failure);
                     responses.set(i, bulkItemResponse);
                     // make sure the request gets never processed again
@@ -359,13 +365,15 @@ protected void doRun() throws Exception {
                     continue;
                 }
                 String concreteIndex = concreteIndices.getConcreteIndex(request.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, request.id(), request.routing()).shardId();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, request.id(),
+                    request.routing()).shardId();
                 List<BulkItemRequest> shardRequests = requestsByShard.computeIfAbsent(shardId, shard -> new ArrayList<>());
                 shardRequests.add(new BulkItemRequest(i, request));
             }
 
             if (requestsByShard.isEmpty()) {
-                listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]),
+                    buildTookInMillis(startTimeNanos)));
                 return;
             }
 
@@ -411,7 +419,8 @@ public void onFailure(Exception e) {
                     }
 
                     private void finishHim() {
-                        listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                        listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]),
+                            buildTookInMillis(startTimeNanos)));
                     }
                 });
             }
@@ -539,7 +548,8 @@ void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListen
                 } else {
                     long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
                     BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
-                    ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener);
+                    ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis,
+                        listener);
                     if (bulkRequest.requests().isEmpty()) {
                         // at this stage, the transport bulk action can't deal with a bulk request with no requests,
                         // so we stop and send an empty response back to the client.
@@ -632,7 +642,8 @@ void markCurrentItemAsFailed(Exception e) {
             // 2) Add a bulk item failure for this request
             // 3) Continue with the next request in the bulk.
             failedSlots.set(currentSlot);
-            BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), e);
+            BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(),
+                indexRequest.id(), e);
             itemResponses.add(new BulkItemResponse(currentSlot, indexRequest.opType(), failure));
         }
 
@@ -645,7 +656,8 @@ static final class IngestBulkResponseListener implements ActionListener<BulkResp
         private final List<BulkItemResponse> itemResponses;
         private final ActionListener<BulkResponse> actionListener;
 
-        IngestBulkResponseListener(long ingestTookInMillis, int[] originalSlots, List<BulkItemResponse> itemResponses, ActionListener<BulkResponse> actionListener) {
+        IngestBulkResponseListener(long ingestTookInMillis, int[] originalSlots, List<BulkItemResponse> itemResponses,
+                                   ActionListener<BulkResponse> actionListener) {
             this.ingestTookInMillis = ingestTookInMillis;
             this.itemResponses = itemResponses;
             this.actionListener = actionListener;
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 32a2d5f056af9..dc07caf63923e 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -97,7 +97,7 @@ public TransportShardBulkAction(Settings settings, TransportService transportSer
     }
 
     @Override
-    protected TransportRequestOptions transportOptions() {
+    protected TransportRequestOptions transportOptions(Settings settings) {
         return BulkAction.INSTANCE.transportOptions(settings);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
index 50d1f4cb8e9c7..ef4adaae4b0dd 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -45,7 +46,8 @@
  * @see org.elasticsearch.client.Client#delete(DeleteRequest)
  * @see org.elasticsearch.client.Requests#deleteRequest(String)
  */
-public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocWriteRequest<DeleteRequest>, CompositeIndicesRequest {
+public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest>
+        implements DocWriteRequest<DeleteRequest>, CompositeIndicesRequest {
 
     private String type;
     private String id;
@@ -83,14 +85,15 @@ public DeleteRequest(String index, String type, String id) {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = super.validate();
-        if (type == null) {
+        if (Strings.isEmpty(type)) {
             validationException = addValidationError("type is missing", validationException);
         }
-        if (id == null) {
+        if (Strings.isEmpty(id)) {
             validationException = addValidationError("id is missing", validationException);
         }
-        if (!versionType.validateVersionForWrites(version)) {
-            validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
+        if (versionType.validateVersionForWrites(version) == false) {
+            validationException = addValidationError("illegal version value [" + version + "] for version type ["
+                + versionType.name() + "]", validationException);
         }
         if (versionType == VersionType.FORCE) {
             validationException = addValidationError("version type [force] may no longer be used", validationException);
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
index 6fdf355c0670c..ee18dfd8d2140 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
@@ -34,6 +34,8 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 /**
  * Explain request encapsulating the explain query and document identifier to get an explanation for.
  */
@@ -152,11 +154,11 @@ public ExplainRequest filteringAlias(AliasFilter filteringAlias) {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = super.validateNonNullIndex();
-        if (type == null) {
-            validationException = ValidateActions.addValidationError("type is missing", validationException);
+        if (Strings.isEmpty(type)) {
+            validationException = addValidationError("type is missing", validationException);
         }
-        if (id == null) {
-            validationException = ValidateActions.addValidationError("id is missing", validationException);
+        if (Strings.isEmpty(id)) {
+            validationException = addValidationError("id is missing", validationException);
         }
         if (query == null) {
             validationException = ValidateActions.addValidationError("query is missing", validationException);
diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index b539f20dd96e4..7e6c18c3e5cfb 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -114,7 +114,8 @@ protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId
                 // Advantage is that we're not opening a second searcher to retrieve the _source. Also
                 // because we are working in the same searcher in engineGetResult we can be sure that a
                 // doc isn't deleted between the initial get and this call.
-                GetResult getResult = context.indexShard().getService().get(result, request.id(), request.type(), request.storedFields(), request.fetchSourceContext());
+                GetResult getResult = context.indexShard().getService().get(result, request.id(), request.type(), request.storedFields(),
+                    request.fetchSourceContext());
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation, getResult);
             } else {
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation);
@@ -134,7 +135,8 @@ protected ExplainResponse newResponse() {
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
         return clusterService.operationRouting().getShards(
-                clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()
+                clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(),
+            request.request().preference()
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
index 7a9b4ae18d39b..998899446e802 100644
--- a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -32,6 +33,8 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 /**
  * A request to get a document (its source) from an index based on its type (optional) and id. Best created using
  * {@link org.elasticsearch.client.Requests#getRequest(String)}.
@@ -91,15 +94,15 @@ public GetRequest(String index, String type, String id) {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = super.validateNonNullIndex();
-        if (type == null) {
-            validationException = ValidateActions.addValidationError("type is missing", validationException);
+        if (Strings.isEmpty(type)) {
+            validationException = addValidationError("type is missing", validationException);
         }
-        if (id == null) {
-            validationException = ValidateActions.addValidationError("id is missing", validationException);
+        if (Strings.isEmpty(id)) {
+            validationException = addValidationError("id is missing", validationException);
         }
-        if (!versionType.validateVersionForReads(version)) {
-            validationException = ValidateActions.addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]",
-                    validationException);
+        if (versionType.validateVersionForReads(version) == false) {
+            validationException = ValidateActions.addValidationError("illegal version value [" + version + "] for version type ["
+                    + versionType.name() + "]", validationException);
         }
         if (versionType == VersionType.FORCE) {
             validationException = ValidateActions.addValidationError("version type [force] may no longer be used", validationException);
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index 6246ca38ef75d..2244e6c158369 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -372,7 +372,8 @@ public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defau
                 currentFieldName = parser.currentName();
             } else if (token == Token.START_ARRAY) {
                 if ("docs".equals(currentFieldName)) {
-                    parseDocuments(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting, allowExplicitIndex);
+                    parseDocuments(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting,
+                        allowExplicitIndex);
                 } else if ("ids".equals(currentFieldName)) {
                     parseIds(parser, this.items, defaultIndex, defaultType, defaultFields, defaultFetchSource, defaultRouting);
                 } else {
@@ -396,7 +397,9 @@ public MultiGetRequest add(@Nullable String defaultIndex, @Nullable String defau
         return this;
     }
 
-    private static void parseDocuments(XContentParser parser, List<Item> items, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, boolean allowExplicitIndex) throws IOException {
+    private static void parseDocuments(XContentParser parser, List<Item> items, @Nullable String defaultIndex, @Nullable String defaultType,
+                                       @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource,
+                                       @Nullable String defaultRouting, boolean allowExplicitIndex) throws IOException {
         String currentFieldName = null;
         Token token;
         while ((token = parser.nextToken()) != Token.END_ARRAY) {
@@ -510,18 +513,22 @@ private static void parseDocuments(XContentParser parser, List<Item> items, @Nul
             } else {
                 aFields = defaultFields;
             }
-            items.add(new Item(index, type, id).routing(routing).storedFields(aFields).parent(parent).version(version).versionType(versionType)
+            items.add(new Item(index, type, id).routing(routing).storedFields(aFields).parent(parent).version(version)
+                    .versionType(versionType)
                     .fetchSourceContext(fetchSourceContext == FetchSourceContext.FETCH_SOURCE ? defaultFetchSource : fetchSourceContext));
         }
     }
 
-    public static void parseIds(XContentParser parser, List<Item> items, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting) throws IOException {
+    public static void parseIds(XContentParser parser, List<Item> items, @Nullable String defaultIndex, @Nullable String defaultType,
+                                @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource,
+                                @Nullable String defaultRouting) throws IOException {
         Token token;
         while ((token = parser.nextToken()) != Token.END_ARRAY) {
             if (!token.isValue()) {
                 throw new IllegalArgumentException("ids array element should only contain ids");
             }
-            items.add(new Item(defaultIndex, defaultType, parser.text()).storedFields(defaultFields).fetchSourceContext(defaultFetchSource).routing(defaultRouting));
+            items.add(new Item(defaultIndex, defaultType, parser.text()).storedFields(defaultFields).fetchSourceContext(defaultFetchSource)
+                .routing(defaultRouting));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index f27012ef8220e..93d1e882d58e6 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -61,14 +61,16 @@ protected boolean resolveIndex(GetRequest request) {
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
         return clusterService.operationRouting()
-                .getShards(clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference());
+                .getShards(clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(),
+                    request.request().preference());
     }
 
     @Override
     protected void resolveRequest(ClusterState state, InternalRequest request) {
         IndexMetaData indexMeta = state.getMetaData().index(request.concreteIndex());
         // update the routing (request#index here is possibly an alias)
-        request.request().routing(state.metaData().resolveIndexRouting(request.request().parent(), request.request().routing(), request.request().index()));
+        request.request().routing(state.metaData().resolveIndexRouting(request.request().parent(), request.request().routing(),
+            request.request().index()));
         // Fail fast on the node that received the request.
         if (request.request().routing() == null && state.getMetaData().routingRequired(request.concreteIndex(), request.request().type())) {
             throw new RoutingMissingException(request.concreteIndex(), request.request().type(), request.request().id());
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 7a7c02ad476e7..46d0b374a59c1 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -86,8 +86,8 @@ protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, Sha
         for (int i = 0; i < request.locations.size(); i++) {
             MultiGetRequest.Item item = request.items.get(i);
             try {
-                GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(), item.version(),
-                    item.versionType(), item.fetchSourceContext());
+                GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(),
+                    item.version(), item.versionType(), item.fetchSourceContext());
                 response.add(request.locations.get(i), new GetResponse(getResult));
             } catch (RuntimeException e) {
                 if (TransportActions.isShardNotAvailableException(e)) {
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 7240b62876c05..57bc50ac36211 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -156,12 +156,14 @@ public ActionRequestValidationException validate() {
         final long resolvedVersion = resolveVersionDefaults();
         if (opType() == OpType.CREATE) {
             if (versionType != VersionType.INTERNAL) {
-                validationException = addValidationError("create operations only support internal versioning. use index instead", validationException);
+                validationException = addValidationError("create operations only support internal versioning. use index instead",
+                    validationException);
                 return validationException;
             }
 
             if (resolvedVersion != Versions.MATCH_DELETED) {
-                validationException = addValidationError("create operations do not support explicit versions. use index instead", validationException);
+                validationException = addValidationError("create operations do not support explicit versions. use index instead",
+                    validationException);
                 return validationException;
             }
         }
@@ -171,7 +173,8 @@ public ActionRequestValidationException validate() {
         }
 
         if (!versionType.validateVersionForWrites(resolvedVersion)) {
-            validationException = addValidationError("illegal version value [" + resolvedVersion + "] for version type [" + versionType.name() + "]", validationException);
+            validationException = addValidationError("illegal version value [" + resolvedVersion + "] for version type ["
+                + versionType.name() + "]", validationException);
         }
 
         if (versionType == VersionType.FORCE) {
@@ -362,7 +365,8 @@ public IndexRequest source(XContentType xContentType, Object... source) {
             throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]");
         }
         if (source.length == 2 && source[0] instanceof BytesReference && source[1] instanceof Boolean) {
-            throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)");
+            throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag"
+                + " was removed, please just use source(BytesReference)");
         }
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
diff --git a/server/src/main/java/org/elasticsearch/action/main/TransportMainAction.java b/server/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
index 368696a9553d9..75c2eee25e4fb 100644
--- a/server/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
@@ -36,6 +36,7 @@
 
 public class TransportMainAction extends HandledTransportAction<MainRequest, MainResponse> {
 
+    private final String nodeName;
     private final ClusterService clusterService;
 
     @Inject
@@ -43,16 +44,16 @@ public TransportMainAction(Settings settings, ThreadPool threadPool, TransportSe
                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                ClusterService clusterService) {
         super(settings, MainAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MainRequest::new);
+        this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.clusterService = clusterService;
     }
 
     @Override
     protected void doExecute(MainRequest request, ActionListener<MainResponse> listener) {
         ClusterState clusterState = clusterService.state();
-        assert Node.NODE_NAME_SETTING.exists(settings);
         final boolean available = clusterState.getBlocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE) == false;
         listener.onResponse(
-            new MainResponse(Node.NODE_NAME_SETTING.get(settings), Version.CURRENT, clusterState.getClusterName(),
+            new MainResponse(nodeName, Version.CURRENT, clusterState.getClusterName(),
                     clusterState.metaData().clusterUUID(), Build.CURRENT, available));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index 772ce0fa4b056..0fa0f238699a6 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -58,7 +58,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice
     private int maxConcurrentSearchRequests = 0;
     private List<SearchRequest> requests = new ArrayList<>();
 
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     /**
      * Add a search request to execute. Note, the order is important, the search response will be returned in the
@@ -288,7 +288,7 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
         }
         return output.toByteArray();
     }
-    
+
     public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException {
         xContentBuilder.startObject();
         if (request.indices() != null) {
diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
index 6cebb73fb4fa6..f08fe1ed57890 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
@@ -28,6 +28,7 @@
  */
 public class MultiSearchRequestBuilder extends ActionRequestBuilder<MultiSearchRequest, MultiSearchResponse, MultiSearchRequestBuilder> {
 
+
     public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction action) {
         super(client, action, new MultiSearchRequest());
     }
@@ -40,7 +41,8 @@ public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction a
      * will not be used (if set).
      */
     public MultiSearchRequestBuilder add(SearchRequest request) {
-        if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+        if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
+            && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
             request.indicesOptions(request().indicesOptions());
         }
 
@@ -53,7 +55,8 @@ public MultiSearchRequestBuilder add(SearchRequest request) {
      * same order as the search requests.
      */
     public MultiSearchRequestBuilder add(SearchRequestBuilder request) {
-        if (request.request().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+        if (request.request().indicesOptions() == SearchRequest.DEFAULT_INDICES_OPTIONS
+            && request().indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
             request.request().indicesOptions(request().indicesOptions());
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 1c67d7cbb6811..fb74b211c9011 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -33,8 +33,6 @@
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
 import org.elasticsearch.common.collect.HppcMaps;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
@@ -68,7 +66,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
-public final class SearchPhaseController extends AbstractComponent {
+public final class SearchPhaseController {
 
     private static final ScoreDoc[] EMPTY_DOCS = new ScoreDoc[0];
 
@@ -76,11 +74,9 @@ public final class SearchPhaseController extends AbstractComponent {
 
     /**
      * Constructor.
-     * @param settings Node settings
      * @param reduceContextFunction A function that builds a context for the reduce of an {@link InternalAggregation}
      */
-    public SearchPhaseController(Settings settings, Function<Boolean, ReduceContext> reduceContextFunction) {
-        super(settings);
+    public SearchPhaseController(Function<Boolean, ReduceContext> reduceContextFunction) {
         this.reduceContextFunction = reduceContextFunction;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 105d5d3aa7b08..8aa849b509ce3 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -91,7 +91,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
 
     private String[] types = Strings.EMPTY_ARRAY;
 
-    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed();
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 199f1104954c0..0eee0ad7ea624 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -27,11 +27,9 @@
 import org.elasticsearch.action.support.HandledTransportAction.ChannelActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
@@ -70,7 +68,7 @@
  * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through
  * transport.
  */
-public class SearchTransportService extends AbstractComponent {
+public class SearchTransportService {
 
     public static final String FREE_CONTEXT_SCROLL_ACTION_NAME = "indices:data/read/search[free_context/scroll]";
     public static final String FREE_CONTEXT_ACTION_NAME = "indices:data/read/search[free_context]";
@@ -88,9 +86,8 @@ public class SearchTransportService extends AbstractComponent {
     private final BiFunction<Transport.Connection, SearchActionListener, ActionListener> responseWrapper;
     private final Map<String, Long> clientConnections = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
 
-    public SearchTransportService(Settings settings, TransportService transportService,
+    public SearchTransportService(TransportService transportService,
                                   BiFunction<Transport.Connection, SearchActionListener, ActionListener> responseWrapper) {
-        super(settings);
         this.transportService = transportService;
         this.responseWrapper = responseWrapper;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java b/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java
index 3e12d0cc84223..c23fe476dcc16 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ActionFilter.java
@@ -22,8 +22,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 
 /**
@@ -47,12 +45,7 @@ <Request extends ActionRequest, Response extends ActionResponse> void apply(Task
      * filter chain. This base class should serve any action filter implementations that doesn't require
      * to apply async filtering logic.
      */
-    abstract class Simple extends AbstractComponent implements ActionFilter {
-
-        protected Simple(Settings settings) {
-            super(settings);
-        }
-
+    abstract class Simple implements ActionFilter {
         @Override
         public final <Request extends ActionRequest, Response extends ActionResponse> void apply(Task task, String action, Request request,
                 ActionListener<Response> listener, ActionFilterChain<Request, Response> chain) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java b/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
index 30d6461ef614b..b87ff9f7ec3bd 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -42,8 +41,7 @@ public class ActiveShardsObserver extends AbstractComponent {
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
 
-    public ActiveShardsObserver(final Settings settings, final ClusterService clusterService, final ThreadPool threadPool) {
-        super(settings);
+    public ActiveShardsObserver(final ClusterService clusterService, final ThreadPool threadPool) {
         this.clusterService = clusterService;
         this.threadPool = threadPool;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java b/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
index 56d5bf206f370..583c34e09641a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
+++ b/server/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.support;
 
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -28,7 +27,7 @@
 /**
  * Helper for dealing with destructive operations and wildcard usage.
  */
-public final class DestructiveOperations extends AbstractComponent {
+public final class DestructiveOperations {
 
     /**
      * Setting which controls whether wildcard usage (*, prefix*, _all) is allowed.
@@ -38,7 +37,6 @@ public final class DestructiveOperations extends AbstractComponent {
     private volatile boolean destructiveRequiresName;
 
     public DestructiveOperations(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         destructiveRequiresName = REQUIRES_NAME_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(REQUIRES_NAME_SETTING, this::setDestructiveRequiresName);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index c707fed6ddf7b..48a479d72b6a7 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -81,7 +81,8 @@ public enum Option {
         IGNORE_ALIASES,
         ALLOW_NO_INDICES,
         FORBID_ALIASES_TO_MULTIPLE_INDICES,
-        FORBID_CLOSED_INDICES;
+        FORBID_CLOSED_INDICES,
+        IGNORE_THROTTLED;
 
         public static final EnumSet<Option> NONE = EnumSet.noneOf(Option.class);
     }
@@ -94,6 +95,9 @@ public enum Option {
         new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED));
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED =
         new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES), EnumSet.of(WildcardStates.OPEN));
+    public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED =
+        new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES, Option.IGNORE_THROTTLED),
+            EnumSet.of(WildcardStates.OPEN));
     public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED =
         new IndicesOptions(EnumSet.of(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES, Option.FORBID_CLOSED_INDICES),
             EnumSet.noneOf(WildcardStates.class));
@@ -242,7 +246,21 @@ public boolean ignoreAliases() {
         return options.contains(Option.IGNORE_ALIASES);
     }
 
+    /**
+     *
+     * @return whether indices that are marked as throttled should be ignored when resolving a wildcard or alias
+     */
+    public boolean ignoreThrottled() {
+        return options.contains(Option.IGNORE_THROTTLED);
+    }
+
     public void writeIndicesOptions(StreamOutput out) throws IOException {
+        EnumSet<Option> options = this.options;
+        // never write this out to a pre7.0 version
+        if (out.getVersion().before(Version.V_6_6_0) && options.contains(Option.IGNORE_THROTTLED)) {
+            options = EnumSet.copyOf(options);
+            options.remove(Option.IGNORE_THROTTLED);
+        }
         if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
             out.writeEnumSet(options);
             out.writeEnumSet(expandWildcards);
@@ -269,16 +287,21 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
         }
     }
 
-    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices) {
-        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, true, false, false);
+    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
+                                             boolean expandToClosedIndices) {
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, true, false, false, false);
     }
 
-    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices, IndicesOptions defaultOptions) {
-        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, defaultOptions.allowAliasesToMultipleIndices(), defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases());
+    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
+                                             boolean expandToClosedIndices, IndicesOptions defaultOptions) {
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices,
+            defaultOptions.allowAliasesToMultipleIndices(), defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases(),
+            defaultOptions.ignoreThrottled());
     }
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
-            boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases) {
+            boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases,
+                                             boolean ignoreThrottled) {
         final Set<Option> opts = new HashSet<>();
         final Set<WildcardStates> wildcards = new HashSet<>();
 
@@ -303,6 +326,9 @@ public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allo
         if (ignoreAliases) {
             opts.add(Option.IGNORE_ALIASES);
         }
+        if (ignoreThrottled) {
+            opts.add(Option.IGNORE_THROTTLED);
+        }
 
         return new IndicesOptions(opts, wildcards);
     }
@@ -312,6 +338,7 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def
                 request.param("expand_wildcards"),
                 request.param("ignore_unavailable"),
                 request.param("allow_no_indices"),
+                request.param("ignore_throttled"),
                 defaultSettings);
     }
 
@@ -320,6 +347,7 @@ public static IndicesOptions fromMap(Map<String, Object> map, IndicesOptions def
                 map.containsKey("expand_wildcards") ? map.get("expand_wildcards") : map.get("expandWildcards"),
                 map.containsKey("ignore_unavailable") ? map.get("ignore_unavailable") : map.get("ignoreUnavailable"),
                 map.containsKey("allow_no_indices") ? map.get("allow_no_indices") : map.get("allowNoIndices"),
+                map.containsKey("ignore_throttled") ? map.get("ignore_throttled") : map.get("ignoreThrottled"),
                 defaultSettings);
     }
 
@@ -330,11 +358,13 @@ public static IndicesOptions fromMap(Map<String, Object> map, IndicesOptions def
     public static boolean isIndicesOptions(String name) {
         return "expand_wildcards".equals(name) || "expandWildcards".equals(name) ||
                 "ignore_unavailable".equals(name) || "ignoreUnavailable".equals(name) ||
+                "ignore_throttled".equals(name) || "ignoreThrottled".equals(name) ||
                 "allow_no_indices".equals(name) || "allowNoIndices".equals(name);
     }
 
-    public static IndicesOptions fromParameters(Object wildcardsString, Object ignoreUnavailableString, Object allowNoIndicesString, IndicesOptions defaultSettings) {
-        if (wildcardsString == null && ignoreUnavailableString == null && allowNoIndicesString == null) {
+    public static IndicesOptions fromParameters(Object wildcardsString, Object ignoreUnavailableString, Object allowNoIndicesString,
+                                                Object ignoreThrottled, IndicesOptions defaultSettings) {
+        if (wildcardsString == null && ignoreUnavailableString == null && allowNoIndicesString == null && ignoreThrottled == null) {
             return defaultSettings;
         }
 
@@ -348,7 +378,8 @@ public static IndicesOptions fromParameters(Object wildcardsString, Object ignor
                 wildcards.contains(WildcardStates.CLOSED),
                 defaultSettings.allowAliasesToMultipleIndices(),
                 defaultSettings.forbidClosedIndices(),
-                defaultSettings.ignoreAliases()
+                defaultSettings.ignoreAliases(),
+                nodeBooleanValue(ignoreThrottled, "ignore_throttled", defaultSettings.ignoreThrottled())
         );
     }
 
@@ -361,6 +392,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         builder.endArray();
         builder.field("ignore_unavailable", ignoreUnavailable());
         builder.field("allow_no_indices", allowNoIndices());
+        builder.field("ignore_throttled", ignoreThrottled());
         return builder;
     }
 
@@ -381,6 +413,15 @@ public static IndicesOptions strictExpandOpenAndForbidClosed() {
         return STRICT_EXPAND_OPEN_FORBID_CLOSED;
     }
 
+    /**
+     * @return indices options that requires every specified index to exist, expands wildcards only to open indices,
+     *         allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
+     *         use of closed indices by throwing an error and ignores indices that are throttled.
+     */
+    public static IndicesOptions strictExpandOpenAndForbidClosedIgnoreThrottled() {
+        return STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED;
+    }
+
     /**
      * @return indices option that requires every specified index to exist, expands wildcards to both open and closed
      * indices and allows that no indices are resolved from wildcard expressions (not returning an error).
@@ -435,6 +476,7 @@ public String toString() {
                 ", allow_aliases_to_multiple_indices=" + allowAliasesToMultipleIndices() +
                 ", forbid_closed_indices=" + forbidClosedIndices() +
                 ", ignore_aliases=" + ignoreAliases() +
+                ", ignore_throttled=" + ignoreThrottled() +
                 ']';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 22edbfca2dc12..63a576f077426 100644
--- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -47,7 +47,7 @@ public abstract class TransportAction<Request extends ActionRequest, Response ex
 
     protected TransportAction(Settings settings, String actionName, ThreadPool threadPool, ActionFilters actionFilters,
                               IndexNameExpressionResolver indexNameExpressionResolver, TaskManager taskManager) {
-        super(settings);
+        // TODO drop settings from ctor
         this.threadPool = threadPool;
         this.actionName = actionName;
         this.filters = actionFilters.filters();
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index 9828dade31a94..4d904c5fd1d5c 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -41,6 +41,11 @@ public abstract class AcknowledgedRequest<Request extends MasterNodeRequest<Requ
     protected AcknowledgedRequest() {
     }
 
+    protected AcknowledgedRequest(StreamInput in) throws IOException {
+        super(in);
+        timeout = in.readTimeValue();
+    }
+
     /**
      * Allows to set the timeout
      * @param timeout timeout as a string (e.g. 1s)
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java
index 8ffecca11ce2f..d85c83190ca4e 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadRequest.java
@@ -31,6 +31,14 @@ public abstract class MasterNodeReadRequest<Request extends MasterNodeReadReques
 
     protected boolean local = false;
 
+    protected MasterNodeReadRequest() {
+    }
+
+    protected MasterNodeReadRequest(StreamInput in) throws IOException {
+        super(in);
+        local = in.readBoolean();
+    }
+
     @SuppressWarnings("unchecked")
     public final Request local(boolean local) {
         this.local = local;
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index 644994c9e2878..ec85b826bd6ac 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -38,6 +38,11 @@ public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Reques
     protected MasterNodeRequest() {
     }
 
+    protected MasterNodeRequest(StreamInput in) throws IOException {
+        super(in);
+        masterNodeTimeout = in.readTimeValue();
+    }
+
     /**
      * A timeout value in case the master has not been discovered yet or disconnected.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index bde70f977591f..9e330d8eb5133 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -70,6 +70,12 @@ protected TransportMasterNodeAction(Settings settings, String actionName, Transp
         this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, request);
     }
 
+    protected TransportMasterNodeAction(Settings settings, String actionName, TransportService transportService,
+                                        ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
+                                        Writeable.Reader<Request> request, IndexNameExpressionResolver indexNameExpressionResolver) {
+        this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, request, indexNameExpressionResolver);
+    }
+
     protected TransportMasterNodeAction(Settings settings, String actionName, boolean canTripCircuitBreaker,
                                         TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
                                         ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
@@ -81,6 +87,17 @@ protected TransportMasterNodeAction(Settings settings, String actionName, boolea
         this.executor = executor();
     }
 
+    protected TransportMasterNodeAction(Settings settings, String actionName, boolean canTripCircuitBreaker,
+                                        TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                        ActionFilters actionFilters, Writeable.Reader<Request> request,
+                                        IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, request,
+            indexNameExpressionResolver);
+        this.transportService = transportService;
+        this.clusterService = clusterService;
+        this.executor = executor();
+    }
+
     protected abstract String executor();
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
index ceb8e1a40ac88..348ef969de96f 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -49,6 +50,13 @@ protected TransportMasterNodeReadAction(Settings settings, String actionName, Tr
         this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,request);
     }
 
+    protected TransportMasterNodeReadAction(Settings settings, String actionName, TransportService transportService,
+                                            ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
+                                            Writeable.Reader<Request> request, IndexNameExpressionResolver indexNameExpressionResolver) {
+        this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, request,
+            indexNameExpressionResolver);
+    }
+
     protected TransportMasterNodeReadAction(Settings settings, String actionName, boolean checkSizeLimit, TransportService transportService,
                                             ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
                                             IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
@@ -57,6 +65,14 @@ protected TransportMasterNodeReadAction(Settings settings, String actionName, bo
         this.forceLocal = FORCE_LOCAL_SETTING.get(settings);
     }
 
+    protected TransportMasterNodeReadAction(Settings settings, String actionName, boolean checkSizeLimit, TransportService transportService,
+                                            ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
+                                            Writeable.Reader<Request> request, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, actionName, checkSizeLimit, transportService, clusterService, threadPool, actionFilters, request,
+            indexNameExpressionResolver);
+        this.forceLocal = FORCE_LOCAL_SETTING.get(settings);
+    }
+
     @Override
     protected final boolean localExecute(Request request) {
         return forceLocal || request.local();
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
index fc14cd38e5681..c012a9c8215ce 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
@@ -35,6 +35,14 @@ public abstract class ClusterInfoRequest<Request extends ClusterInfoRequest<Requ
 
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
 
+    public ClusterInfoRequest() {
+
+    }
+
+    public ClusterInfoRequest(final StreamInput in) throws IOException {
+        readFrom(in);
+    }
+
     @Override
     @SuppressWarnings("unchecked")
     public Request indices(String... indices) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index aeb357e2482fd..a8abc0451ea66 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -143,7 +143,7 @@ protected TransportReplicationAction(Settings settings, String actionName, Trans
         this.transportReplicaAction = actionName + "[r]";
         registerRequestHandlers(actionName, transportService, request, replicaRequest, executor);
 
-        this.transportOptions = transportOptions();
+        this.transportOptions = transportOptions(settings);
 
         this.syncGlobalCheckpointAfterOperation = syncGlobalCheckpointAfterOperation;
     }
@@ -232,7 +232,7 @@ protected boolean resolveIndex() {
         return true;
     }
 
-    protected TransportRequestOptions transportOptions() {
+    protected TransportRequestOptions transportOptions(Settings settings) {
         return TransportRequestOptions.EMPTY;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 137b98bceeec0..c3eb9f7fbda22 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -62,8 +61,7 @@ public class UpdateHelper extends AbstractComponent {
 
     private final ScriptService scriptService;
 
-    public UpdateHelper(Settings settings, ScriptService scriptService) {
-        super(settings);
+    public UpdateHelper(ScriptService scriptService) {
         this.scriptService = scriptService;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index 2dcd35dfb36b9..a6958ba9e355b 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -28,8 +28,11 @@
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -56,6 +59,9 @@
 public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
         implements DocWriteRequest<UpdateRequest>, WriteRequest<UpdateRequest>, ToXContentObject {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER =
+        new DeprecationLogger(Loggers.getLogger(UpdateRequest.class));
+
     private String type;
     private String id;
     @Nullable
@@ -106,10 +112,10 @@ public ActionRequestValidationException validate() {
         if(upsertRequest != null && upsertRequest.version() != Versions.MATCH_ANY) {
             validationException = addValidationError("can't provide version in upsert request", validationException);
         }
-        if (type == null) {
+        if (Strings.isEmpty(type)) {
             validationException = addValidationError("type is missing", validationException);
         }
-        if (id == null) {
+        if (Strings.isEmpty(id)) {
             validationException = addValidationError("id is missing", validationException);
         }
 
@@ -726,7 +732,22 @@ public UpdateRequest fromXContent(XContentParser parser) throws IOException {
             return this;
         }
         String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+        int depth = 0;
+        while (token != null) {
+            token = parser.nextToken();
+            // have to track depth to avoid premature end of parsing due to nested objects as
+            // update request could have item ( script / scripted_upsert / upsert / doc etc ) at any nested level
+            if (token == XContentParser.Token.START_OBJECT) {
+                depth++;
+            } else if (token == XContentParser.Token.END_OBJECT) {
+                currentFieldName = null;
+                if (depth == 0) {
+                    break;
+                }
+                depth--;
+                continue;
+            }
+
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if ("script".equals(currentFieldName)) {
@@ -746,6 +767,7 @@ public UpdateRequest fromXContent(XContentParser parser) throws IOException {
             } else if ("detect_noop".equals(currentFieldName)) {
                 detectNoop(parser.booleanValue());
             } else if ("fields".equals(currentFieldName)) {
+                DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
                 List<Object> fields = null;
                 if (token == XContentParser.Token.START_ARRAY) {
                     fields = (List) parser.list();
@@ -757,6 +779,14 @@ public UpdateRequest fromXContent(XContentParser parser) throws IOException {
                 }
             } else if ("_source".equals(currentFieldName)) {
                 fetchSourceContext = FetchSourceContext.fromXContent(parser);
+            } else {
+                DEPRECATION_LOGGER.deprecated("Unknown field [{}] used in {} which has no value and will not be accepted in future",
+                    currentFieldName, UpdateRequest.class.getSimpleName());
+            }
+
+            // copyCurrentStructure / SomeObject.fromXContent moves current token to END_OBJECT
+            if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+                depth--;
             }
         }
         if (script != null) {
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
index 5ba187013e79f..c2b4222d86ce2 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.rest.action.document.RestUpdateAction;
 import org.elasticsearch.script.Script;
 
 import java.util.Map;
@@ -39,7 +38,7 @@
 public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<UpdateRequest, UpdateResponse, UpdateRequestBuilder>
         implements WriteRequestBuilder<UpdateRequestBuilder> {
     private static final DeprecationLogger DEPRECATION_LOGGER =
-        new DeprecationLogger(Loggers.getLogger(RestUpdateAction.class));
+        new DeprecationLogger(Loggers.getLogger(UpdateRequestBuilder.class));
 
     public UpdateRequestBuilder(ElasticsearchClient client, UpdateAction action) {
         super(client, action, new UpdateRequest());
diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 68551bb5a5826..111c1fd96d90e 100644
--- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -345,12 +345,13 @@
 
 public abstract class AbstractClient extends AbstractComponent implements Client {
 
+    protected final Settings settings;
     private final ThreadPool threadPool;
     private final Admin admin;
     private final ThreadedActionListener.Wrapper threadedWrapper;
 
     public AbstractClient(Settings settings, ThreadPool threadPool) {
-        super(settings);
+        this.settings = settings;
         this.threadPool = threadPool;
         this.admin = new Admin(this);
         this.threadedWrapper = new ThreadedActionListener.Wrapper(logger, settings, threadPool);
diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 0cfc1f5004ce8..b36fe45599590 100644
--- a/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -124,21 +124,20 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
 
     TransportClientNodesService(Settings settings, TransportService transportService,
                                        ThreadPool threadPool, TransportClient.HostFailureListener hostFailureListener) {
-        super(settings);
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.transportService = transportService;
         this.threadPool = threadPool;
         this.minCompatibilityVersion = Version.CURRENT.minimumCompatibilityVersion();
 
-        this.nodesSamplerInterval = TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(this.settings);
-        this.pingTimeout = TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.get(this.settings).millis();
-        this.ignoreClusterName = TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(this.settings);
+        this.nodesSamplerInterval = TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(settings);
+        this.pingTimeout = TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.get(settings).millis();
+        this.ignoreClusterName = TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(settings);
 
         if (logger.isDebugEnabled()) {
             logger.debug("node_sampler_interval[{}]", nodesSamplerInterval);
         }
 
-        if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(this.settings)) {
+        if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(settings)) {
             this.nodesSampler = new SniffNodesSampler();
         } else {
             this.nodesSampler = new SimpleNodeSampler();
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 98b0d3b330aee..9bb608cc7ac16 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -108,11 +108,11 @@ public class ClusterModule extends AbstractModule {
     public ClusterModule(Settings settings, ClusterService clusterService, List<ClusterPlugin> clusterPlugins,
                          ClusterInfoService clusterInfoService) {
         this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
-        this.allocationDeciders = new AllocationDeciders(settings, deciderList);
+        this.allocationDeciders = new AllocationDeciders(deciderList);
         this.shardsAllocator = createShardsAllocator(settings, clusterService.getClusterSettings(), clusterPlugins);
         this.clusterService = clusterService;
         this.indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
-        this.allocationService = new AllocationService(settings, allocationDeciders, shardsAllocator, clusterInfoService);
+        this.allocationService = new AllocationService(allocationDeciders, shardsAllocator, clusterInfoService);
     }
 
     public static Map<String, Supplier<ClusterState.Custom>> getClusterStateCustomSuppliers(List<ClusterPlugin> clusterPlugins) {
@@ -222,16 +222,16 @@ public static Collection<AllocationDecider> createAllocationDeciders(Settings se
                                                                          List<ClusterPlugin> clusterPlugins) {
         // collect deciders by class so that we can detect duplicates
         Map<Class, AllocationDecider> deciders = new LinkedHashMap<>();
-        addAllocationDecider(deciders, new MaxRetryAllocationDecider(settings));
-        addAllocationDecider(deciders, new ResizeAllocationDecider(settings));
-        addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider(settings));
-        addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider(settings));
+        addAllocationDecider(deciders, new MaxRetryAllocationDecider());
+        addAllocationDecider(deciders, new ResizeAllocationDecider());
+        addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider());
+        addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider());
         addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new EnableAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new NodeVersionAllocationDecider(settings));
-        addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider(settings));
-        addAllocationDecider(deciders, new RestoreInProgressAllocationDecider(settings));
+        addAllocationDecider(deciders, new NodeVersionAllocationDecider());
+        addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider());
+        addAllocationDecider(deciders, new RestoreInProgressAllocationDecider());
         addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings));
         addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
diff --git a/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
index 8e705a2406301..c6632bd524a3a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
@@ -19,19 +19,12 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
-
 /**
  * ClusterInfoService that provides empty maps for disk usage and shard sizes
  */
-public class EmptyClusterInfoService extends AbstractComponent implements ClusterInfoService {
+public class EmptyClusterInfoService implements ClusterInfoService {
     public static final EmptyClusterInfoService INSTANCE = new EmptyClusterInfoService();
 
-    private EmptyClusterInfoService() {
-        super(Settings.EMPTY);
-    }
-
     @Override
     public ClusterInfo getClusterInfo() {
         return ClusterInfo.EMPTY;
diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index a22c2099106b7..184cbcdf859d5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -88,7 +88,6 @@ public class InternalClusterInfoService extends AbstractComponent
 
     public InternalClusterInfoService(Settings settings, ClusterService clusterService, ThreadPool threadPool, NodeClient client,
                                       Consumer<ClusterInfo> listener) {
-        super(settings);
         this.leastAvailableSpaceUsages = ImmutableOpenMap.of();
         this.mostAvailableSpaceUsages = ImmutableOpenMap.of();
         this.shardRoutingToDataPath = ImmutableOpenMap.of();
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
index 56311455a0ee8..770c6bca26b2f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.IndicesAdminClient;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -39,7 +38,7 @@
  * Called by shards in the cluster when their mapping was dynamically updated and it needs to be updated
  * in the cluster state meta data (and broadcast to all members).
  */
-public class MappingUpdatedAction extends AbstractComponent {
+public class MappingUpdatedAction {
 
     public static final Setting<TimeValue> INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING =
         Setting.positiveTimeSetting("indices.mapping.dynamic_timeout", TimeValue.timeValueSeconds(30),
@@ -50,7 +49,6 @@ public class MappingUpdatedAction extends AbstractComponent {
 
     @Inject
     public MappingUpdatedAction(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.dynamicMappingUpdateTimeout = INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING, this::setDynamicMappingUpdateTimeout);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
index 1af5eeea6df90..7b79823380c2e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
@@ -47,8 +46,7 @@ public class NodeMappingRefreshAction extends AbstractComponent {
     private final MetaDataMappingService metaDataMappingService;
 
     @Inject
-    public NodeMappingRefreshAction(Settings settings, TransportService transportService, MetaDataMappingService metaDataMappingService) {
-        super(settings);
+    public NodeMappingRefreshAction(TransportService transportService, MetaDataMappingService metaDataMappingService) {
         this.transportService = transportService;
         this.metaDataMappingService = metaDataMappingService;
         transportService.registerRequestHandler(ACTION_NAME,
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 59a85480e1978..2b481015ca0ee 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.discovery.Discovery;
@@ -88,9 +87,8 @@ public class ShardStateAction extends AbstractComponent {
     private final ConcurrentMap<FailedShardEntry, CompositeListener> remoteFailedShardsCache = ConcurrentCollections.newConcurrentMap();
 
     @Inject
-    public ShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService,
+    public ShardStateAction(ClusterService clusterService, TransportService transportService,
                             AllocationService allocationService, RoutingService routingService, ThreadPool threadPool) {
-        super(settings);
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index 766b35307cdd2..c9258806d51db 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -22,8 +22,6 @@
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -43,12 +41,7 @@
  * Validator for an alias, to be used before adding an alias to the index metadata
  * and make sure the alias is valid
  */
-public class AliasValidator extends AbstractComponent {
-
-    public AliasValidator(Settings settings) {
-        super(settings);
-    }
-
+public class AliasValidator {
     /**
      * Allows to validate an {@link org.elasticsearch.action.admin.indices.alias.Alias} and make sure
      * it's valid before it gets added to the index metadata. Doesn't validate the alias filter.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java
index 2032c2f4ef3ba..9cbf46b6afcbc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolver.java
@@ -19,9 +19,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -33,14 +31,10 @@
  * Resolves cluster names from an expression. The expression must be the exact match of a cluster
  * name or must be a wildcard expression.
  */
-public final class ClusterNameExpressionResolver extends AbstractComponent {
+public final class ClusterNameExpressionResolver {
 
     private final WildcardExpressionResolver wildcardResolver = new WildcardExpressionResolver();
 
-    public ClusterNameExpressionResolver(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Resolves the provided cluster expression to matching cluster names. This method only
      * supports exact or wildcard matches.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index cb3fed0caf215..ce15991646d77 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -35,6 +34,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.joda.time.DateTimeZone;
@@ -54,18 +54,17 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
-public class IndexNameExpressionResolver extends AbstractComponent {
-
+public class IndexNameExpressionResolver {
     private final List<ExpressionResolver> expressionResolvers;
     private final DateMathExpressionResolver dateMathExpressionResolver;
 
     public IndexNameExpressionResolver(Settings settings) {
-        super(settings);
+        dateMathExpressionResolver = new DateMathExpressionResolver(settings);
         expressionResolvers = Arrays.asList(
-                dateMathExpressionResolver = new DateMathExpressionResolver(settings),
-                new WildcardExpressionResolver()
-        );
+                dateMathExpressionResolver,
+                new WildcardExpressionResolver());
     }
+        
 
     /**
      * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options
@@ -152,6 +151,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         if (indexExpressions == null || indexExpressions.length == 0) {
             indexExpressions = new String[]{MetaData.ALL};
         }
+        Set<String> originalIndexExpression = Sets.newHashSet(indexExpressions);
         MetaData metaData = context.getState().metaData();
         IndicesOptions options = context.getOptions();
         final boolean failClosed = options.forbidClosedIndices() && options.ignoreUnavailable() == false;
@@ -201,7 +201,9 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                         " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" +
                         " indices without one being designated as a write index");
                 }
-                concreteIndices.add(writeIndex.getIndex());
+                if (addIndex(writeIndex, context, originalIndexExpression)) {
+                    concreteIndices.add(writeIndex.getIndex());
+                }
             } else {
                 if (aliasOrIndex.getIndices().size() > 1 && !options.allowAliasesToMultipleIndices()) {
                     String[] indexNames = new String[aliasOrIndex.getIndices().size()];
@@ -218,12 +220,14 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                         if (failClosed) {
                             throw new IndexClosedException(index.getIndex());
                         } else {
-                            if (options.forbidClosedIndices() == false) {
+                            if (options.forbidClosedIndices() == false && addIndex(index, context, originalIndexExpression)) {
                                 concreteIndices.add(index.getIndex());
                             }
                         }
                     } else if (index.getState() == IndexMetaData.State.OPEN) {
-                        concreteIndices.add(index.getIndex());
+                        if (addIndex(index, context, originalIndexExpression)) {
+                            concreteIndices.add(index.getIndex());
+                        }
                     } else {
                         throw new IllegalStateException("index state [" + index.getState() + "] not supported");
                     }
@@ -239,6 +243,15 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         return concreteIndices.toArray(new Index[concreteIndices.size()]);
     }
 
+    private static boolean addIndex(IndexMetaData metaData, Context context, Set<String> originalIndices) {
+        if (context.options.ignoreThrottled()) {
+            if (originalIndices.contains(metaData.getIndex().getName()) == false) {
+                return IndexSettings.INDEX_SEARCH_THROTTLED.get(metaData.getSettings()) == false;
+            }
+        }
+        return true;
+    }
+
     private static IllegalArgumentException aliasesNotSupportedException(String expression) {
         return new IllegalArgumentException("The provided expression [" + expression + "] matches an " +
                 "alias, specify the corresponding concrete indices instead.");
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 9be417941ca4a..892ce3abbf311 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -105,6 +105,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
+    private final Settings settings;
     private final ClusterService clusterService;
     private final IndicesService indicesService;
     private final AllocationService allocationService;
@@ -126,14 +127,14 @@ public MetaDataCreateIndexService(
             final ThreadPool threadPool,
             final NamedXContentRegistry xContentRegistry,
             final boolean forbidPrivateIndexSettings) {
-        super(settings);
+        this.settings = settings;
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.allocationService = allocationService;
         this.aliasValidator = aliasValidator;
         this.env = env;
         this.indexScopedSettings = indexScopedSettings;
-        this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool);
         this.xContentRegistry = xContentRegistry;
         this.forbidPrivateIndexSettings = forbidPrivateIndexSettings;
     }
@@ -169,7 +170,8 @@ public static void validateIndexOrAliasName(String index, BiFunction<String, Str
         }
         if (index.contains(":")) {
             deprecationLogger.deprecated("index or alias name [" + index +
-                            "] containing ':' is deprecated and will not be supported in Elasticsearch 7.0+");
+                            "] containing ':' is deprecated. Elasticsearch 7.x will read, " +
+                            "but not allow creation of new indices containing ':'");
         }
         if (index.charAt(0) == '_' || index.charAt(0) == '-' || index.charAt(0) == '+') {
             throw exceptionCtor.apply(index, "must not start with '_', '-', or '+'");
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index a2212b5c3f01c..39563ca7037d9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -48,14 +48,14 @@
  * Deletes indices.
  */
 public class MetaDataDeleteIndexService extends AbstractComponent {
-
+    private final Settings settings;
     private final ClusterService clusterService;
 
     private final AllocationService allocationService;
 
     @Inject
     public MetaDataDeleteIndexService(Settings settings, ClusterService clusterService, AllocationService allocationService) {
-        super(settings);
+        this.settings = settings;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
index 06eb008d20773..e7746f0f39a9f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
@@ -29,9 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -54,7 +52,7 @@
 /**
  * Service responsible for submitting add and remove aliases requests
  */
-public class MetaDataIndexAliasesService extends AbstractComponent {
+public class MetaDataIndexAliasesService {
 
     private final ClusterService clusterService;
 
@@ -67,9 +65,8 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
     private final NamedXContentRegistry xContentRegistry;
 
     @Inject
-    public MetaDataIndexAliasesService(Settings settings, ClusterService clusterService, IndicesService indicesService,
+    public MetaDataIndexAliasesService(ClusterService clusterService, IndicesService indicesService,
             AliasValidator aliasValidator, MetaDataDeleteIndexService deleteIndexService, NamedXContentRegistry xContentRegistry) {
-        super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.aliasValidator = aliasValidator;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
index 6badfd6ef586a..38d83b398856e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
@@ -42,7 +42,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
@@ -76,15 +75,14 @@ public class MetaDataIndexStateService extends AbstractComponent {
     private final ActiveShardsObserver activeShardsObserver;
 
     @Inject
-    public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService,
+    public MetaDataIndexStateService(ClusterService clusterService, AllocationService allocationService,
                                      MetaDataIndexUpgradeService metaDataIndexUpgradeService,
                                      IndicesService indicesService, ThreadPool threadPool) {
-        super(settings);
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.metaDataIndexUpgradeService = metaDataIndexUpgradeService;
-        this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool);
     }
 
     public void closeIndex(final CloseIndexClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 953d964ff08e3..63798c67e8165 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -71,11 +71,10 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
     private final NamedXContentRegistry xContentRegistry;
 
     @Inject
-    public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService,
+    public MetaDataIndexTemplateService(ClusterService clusterService,
                                         MetaDataCreateIndexService metaDataCreateIndexService,
                                         AliasValidator aliasValidator, IndicesService indicesService,
                                         IndexScopedSettings indexScopedSettings, NamedXContentRegistry xContentRegistry) {
-        super(settings);
         this.clusterService = clusterService;
         this.aliasValidator = aliasValidator;
         this.indicesService = indicesService;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index b18286f269c82..29f2c1c911888 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -53,6 +53,7 @@
  */
 public class MetaDataIndexUpgradeService extends AbstractComponent {
 
+    private final Settings settings;
     private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
@@ -61,7 +62,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
                                        IndexScopedSettings indexScopedSettings,
                                        Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
-        super(settings);
+        this.settings = settings;
         this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index eb9c87fd86300..4f740e1028392 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
@@ -68,8 +67,7 @@ public class MetaDataMappingService extends AbstractComponent {
 
 
     @Inject
-    public MetaDataMappingService(Settings settings, ClusterService clusterService, IndicesService indicesService) {
-        super(settings);
+    public MetaDataMappingService(ClusterService clusterService, IndicesService indicesService) {
         this.clusterService = clusterService;
         this.indicesService = indicesService;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index b2bc082f2846b..2284d507afa2c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -75,9 +75,8 @@ public class MetaDataUpdateSettingsService extends AbstractComponent {
     private final ThreadPool threadPool;
 
     @Inject
-    public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService,
+    public MetaDataUpdateSettingsService(ClusterService clusterService, AllocationService allocationService,
                                          IndexScopedSettings indexScopedSettings, IndicesService indicesService, ThreadPool threadPool) {
-        super(settings);
         this.clusterService = clusterService;
         this.threadPool = threadPool;
         this.allocationService = allocationService;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
index 1f042b4c576b7..9026d26a11fd5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -75,9 +74,8 @@ public class TemplateUpgradeService extends AbstractComponent implements Cluster
 
     private ImmutableOpenMap<String, IndexTemplateMetaData> lastTemplateMetaData;
 
-    public TemplateUpgradeService(Settings settings, Client client, ClusterService clusterService, ThreadPool threadPool,
+    public TemplateUpgradeService(Client client, ClusterService clusterService, ThreadPool threadPool,
                                   Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
-        super(settings);
         this.client = client;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 479a1461f37ae..215b247bd97ea 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -59,7 +59,6 @@ public class OperationRouting extends AbstractComponent {
     private boolean useAdaptiveReplicaSelection;
 
     public OperationRouting(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.awarenessAttributes = AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings);
         this.useAdaptiveReplicaSelection = USE_ADAPTIVE_REPLICA_SELECTION_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 8c3225dc77fd6..0bc94a93cc59a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;
 
 import java.util.ArrayList;
@@ -69,16 +68,15 @@ public class AllocationService extends AbstractComponent {
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
 
-    public AllocationService(Settings settings, AllocationDeciders allocationDeciders,
+    public AllocationService(AllocationDeciders allocationDeciders,
                              GatewayAllocator gatewayAllocator,
                              ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
-        this(settings, allocationDeciders, shardsAllocator, clusterInfoService);
+        this(allocationDeciders, shardsAllocator, clusterInfoService);
         setGatewayAllocator(gatewayAllocator);
     }
 
-    public AllocationService(Settings settings, AllocationDeciders allocationDeciders,
+    public AllocationService(AllocationDeciders allocationDeciders,
                              ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
-        super(settings);
         this.allocationDeciders = allocationDeciders;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 0ddf3ef1529b3..f2447a9c4e51b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -54,7 +54,6 @@ public class DiskThresholdMonitor extends AbstractComponent {
 
     public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterStateSupplier, ClusterSettings clusterSettings,
                                 Client client) {
-        super(settings);
         this.clusterStateSupplier = clusterStateSupplier;
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.client = client;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 4ddeb8de0737c..597e904e24ec6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -97,7 +97,6 @@ public BalancedShardsAllocator(Settings settings) {
 
     @Inject
     public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         setWeightFunction(INDEX_BALANCE_FACTOR_SETTING.get(settings), SHARD_BALANCE_FACTOR_SETTING.get(settings));
         setThreshold(THRESHOLD_SETTING.get(settings));
         clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, SHARD_BALANCE_FACTOR_SETTING, this::setWeightFunction);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java
index 12cac56e11a44..3ae86d60bd98c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java
@@ -19,13 +19,12 @@
 
 package org.elasticsearch.cluster.routing.allocation.decider;
 
+import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * {@link AllocationDecider} is an abstract base class that allows to make
@@ -33,15 +32,6 @@
  * basis.
  */
 public abstract class AllocationDecider extends AbstractComponent {
-
-    /**
-     * Initializes a new {@link AllocationDecider}
-     * @param settings {@link Settings} used by this {@link AllocationDecider}
-     */
-    protected AllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Returns a {@link Decision} whether the given shard routing can be
      * re-balanced to the given allocation. The default is
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
index ee7f761d65d97..7f91be340fd79 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -38,8 +37,7 @@ public class AllocationDeciders extends AllocationDecider {
 
     private final Collection<AllocationDecider> allocations;
 
-    public AllocationDeciders(Settings settings, Collection<AllocationDecider> allocations) {
-        super(settings);
+    public AllocationDeciders(Collection<AllocationDecider> allocations) {
         this.allocations = Collections.unmodifiableCollection(allocations);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index 6105c732d5511..8994bbf1e45bd 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -92,7 +92,6 @@ public class AwarenessAllocationDecider extends AllocationDecider {
     private volatile Map<String, List<String>> forcedAwarenessAttributes;
 
     public AwarenessAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.awarenessAttributes = CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, this::setAwarenessAttributes);
         setForcedAwarenessAttributes(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.get(settings));
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
index ea945c23c728c..1ea369c75d9bc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
@@ -91,7 +91,6 @@ public String toString() {
     private volatile ClusterRebalanceType type;
 
     public ClusterRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         try {
             type = CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get(settings);
         } catch (IllegalStateException e) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
index 15456ec3e1133..a11b3dcf102f8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
@@ -48,7 +48,6 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider {
     private volatile int clusterConcurrentRebalance;
 
     public ConcurrentRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.clusterConcurrentRebalance = CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.get(settings);
         logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance);
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index a7426d3e551b5..9676eaf4df1c3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -73,7 +73,6 @@ public class DiskThresholdDecider extends AllocationDecider {
     private final DiskThresholdSettings diskThresholdSettings;
 
     public DiskThresholdDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index d0fbe6761b290..8a72fe8cb49a9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -80,7 +80,6 @@ public class EnableAllocationDecider extends AllocationDecider {
     private volatile Allocation enableAllocation;
 
     public EnableAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.enableAllocation = CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get(settings);
         this.enableRebalance = CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, this::setEnableAllocation);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index ed2d5384fa704..053d696f6768c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -97,7 +97,6 @@ public class FilterAllocationDecider extends AllocationDecider {
     private volatile DiscoveryNodeFilters clusterExcludeFilters;
 
     public FilterAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         setClusterRequireFilters(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING.getAsMap(settings));
         setClusterExcludeFilters(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getAsMap(settings));
         setClusterIncludeFilters(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING.getAsMap(settings));
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
index 4c580509e9229..708482feae7d4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * An allocation decider that prevents shards from being allocated on any node if the shards allocation has been retried N times without
@@ -42,15 +41,6 @@ public class MaxRetryAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "max_retry";
 
-    /**
-     * Initializes a new {@link MaxRetryAllocationDecider}
-     *
-     * @param settings {@link Settings} used by this {@link AllocationDecider}
-     */
-    public MaxRetryAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
         final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
index f2df6d3196dd0..e2817eb87a7d7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * An allocation decider that prevents relocation or allocation from nodes
@@ -38,10 +37,6 @@ public class NodeVersionAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "node_version";
 
-    public NodeVersionAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         if (shardRouting.primary()) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
index c4cd2ecf50dda..3d890067bbf3f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * Only allow rebalancing when all shards are active within the shard replication group.
@@ -30,10 +29,6 @@ public class RebalanceOnlyWhenActiveAllocationDecider extends AllocationDecider
 
     public static final String NAME = "rebalance_only_when_active";
 
-    public RebalanceOnlyWhenActiveAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
         if (!allocation.routingNodes().allReplicasActive(shardRouting.shardId(), allocation.metaData())) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
index 4cceb1cc161c3..2bc81b84570b2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * An allocation strategy that only allows for a replica to be allocated when the primary is active.
@@ -31,10 +30,6 @@ public class ReplicaAfterPrimaryActiveAllocationDecider extends AllocationDecide
 
     private static final String NAME = "replica_after_primary_active";
 
-    public ReplicaAfterPrimaryActiveAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
         return canAllocate(shardRouting, allocation);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java
index 8babcd5484f6c..d58a625c6edc3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java
@@ -26,11 +26,9 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
-
 /**
  * An allocation decider that ensures we allocate the shards of a target index for resize operations next to the source primaries
  */
@@ -38,15 +36,6 @@ public class ResizeAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "resize";
 
-    /**
-     * Initializes a new {@link ResizeAllocationDecider}
-     *
-     * @param settings {@link Settings} used by this {@link AllocationDecider}
-     */
-    public ResizeAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
         return canAllocate(shardRouting, null, allocation);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
index 3fefd4e0abba4..63971ca46e468 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDecider.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.Snapshot;
 
 /**
@@ -35,16 +34,6 @@ public class RestoreInProgressAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "restore_in_progress";
 
-    /**
-     * Creates a new {@link RestoreInProgressAllocationDecider} instance from
-     * given settings
-     *
-     * @param settings {@link Settings} to use
-     */
-    public RestoreInProgressAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     @Override
     public Decision canAllocate(final ShardRouting shardRouting, final RoutingNode node, final RoutingAllocation allocation) {
         return canAllocate(shardRouting, allocation);
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index cc2d488974baa..2961b3faaf4dc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -53,7 +53,6 @@ public class SameShardAllocationDecider extends AllocationDecider {
     private volatile boolean sameHost;
 
     public SameShardAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.sameHost = CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING, this::setSameHost);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index 398ee0a17ad54..1c0a0c0ef0a6b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -73,8 +73,10 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
         Setting.intSetting("cluster.routing.allocation.total_shards_per_node", -1,  -1,
             Property.Dynamic, Property.NodeScope);
 
+    private final Settings settings;
+
     public ShardsLimitAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
+        this.settings = settings;
         this.clusterShardLimit = CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING, this::setClusterShardLimit);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
index eb4cc0c4420ff..7eb1b882d1ffe 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * This {@link org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider} prevents shards that
@@ -33,16 +32,6 @@ public class SnapshotInProgressAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "snapshot_in_progress";
 
-    /**
-     * Creates a new {@link org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider} instance from
-     * given settings
-     *
-     * @param settings {@link org.elasticsearch.common.settings.Settings} to use
-     */
-    public SnapshotInProgressAllocationDecider(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Returns a {@link Decision} whether the given shard routing can be
      * re-balanced to the given allocation. The default is
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 7821ad11a52cd..0d67cd6071f08 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -81,7 +81,6 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
     private volatile int concurrentOutgoingRecoveries;
 
     public ThrottlingAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         this.primariesInitialRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.get(settings);
         concurrentIncomingRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.get(settings);
         concurrentOutgoingRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.get(settings);
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index a3e7f0c8a5833..439a2c534a38f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -137,7 +137,7 @@ protected synchronized void doStart() {
         addListener(localNodeMasterListeners);
         threadPoolExecutor = EsExecutors.newSinglePrioritizing(
                 nodeName + "/" + CLUSTER_UPDATE_THREAD_NAME,
-                daemonThreadFactory(settings, CLUSTER_UPDATE_THREAD_NAME),
+                daemonThreadFactory(nodeName, CLUSTER_UPDATE_THREAD_NAME),
                 threadPool.getThreadContext(),
                 threadPool.scheduler());
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index de66cbe6cb6a0..f4c91b2781c71 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -56,6 +56,11 @@ public class ClusterService extends AbstractLifecycleComponent {
     public static final org.elasticsearch.common.settings.Setting.AffixSetting<String> USER_DEFINED_META_DATA =
         Setting.prefixKeySetting("cluster.metadata.", (key) -> Setting.simpleString(key, Property.Dynamic, Property.NodeScope));
 
+    /**
+     * The node's settings.
+     */
+    private final Settings settings;
+
     private final ClusterName clusterName;
 
     private final OperationRouting operationRouting;
@@ -69,6 +74,7 @@ public class ClusterService extends AbstractLifecycleComponent {
     public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool,
                           Map<String, Supplier<ClusterState.Custom>> initialClusterStateCustoms) {
         super(settings);
+        this.settings = settings;
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.masterService = new MasterService(nodeName, settings, threadPool);
         this.operationRouting = new OperationRouting(settings, clusterSettings);
@@ -216,6 +222,9 @@ public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
 
+    /**
+     * The node's settings.
+     */
     public Settings getSettings() {
         return settings;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
index 86fc3b54ca6f4..cbef687fc410a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
@@ -109,7 +109,7 @@ protected synchronized void doStart() {
         Objects.requireNonNull(clusterStateSupplier, "please set a cluster state supplier before starting");
         threadPoolExecutor = EsExecutors.newSinglePrioritizing(
                 nodeName + "/" + MASTER_UPDATE_THREAD_NAME,
-                daemonThreadFactory(settings, MASTER_UPDATE_THREAD_NAME),
+                daemonThreadFactory(nodeName, MASTER_UPDATE_THREAD_NAME),
                 threadPool.getThreadContext(),
                 threadPool.scheduler());
         taskBatcher = new Batcher(logger, threadPoolExecutor);
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
index c49143edb446e..cffdf0f4507c0 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
@@ -42,7 +42,6 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
     private final boolean readOnly;
 
     public FsBlobStore(Settings settings, Path path) throws IOException {
-        super(settings);
         this.path = path;
         this.readOnly = settings.getAsBoolean("readonly", false);
         if (!this.readOnly) {
diff --git a/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java b/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
index bbcb4b9ec4209..1e0310c024736 100644
--- a/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
+++ b/server/src/main/java/org/elasticsearch/common/component/AbstractComponent.java
@@ -21,15 +21,16 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 
+/**
+ * @deprecated declare your own logger
+ */
+@Deprecated
 public abstract class AbstractComponent {
 
     protected final Logger logger;
-    protected final Settings settings;
 
-    public AbstractComponent(Settings settings) {
+    public AbstractComponent() {
         this.logger = LogManager.getLogger(getClass());
-        this.settings = settings;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
index 19f17f6e81160..cee7f3a5f42cf 100644
--- a/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
+++ b/server/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
@@ -32,7 +32,7 @@ public abstract class AbstractLifecycleComponent extends AbstractComponent imple
     private final List<LifecycleListener> listeners = new CopyOnWriteArrayList<>();
 
     protected AbstractLifecycleComponent(Settings settings) {
-        super(settings);
+        // TODO drop settings from ctor
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
index 28df6206e9bc5..9a5ab8730835a 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -76,15 +76,6 @@ public static Logger getLogger(Logger parentLogger, String s) {
         return ESLoggerFactory.getLogger(prefix, parentLogger.getName() + s);
     }
 
-    /**
-     * Get or build a logger.
-     * @deprecated Prefer {@link LogManager#getLogger}
-     */
-    @Deprecated
-    public static Logger getLogger(Class<?> clazz) {
-        return ESLoggerFactory.getLogger(null, clazz);
-    }
-
     private static String formatPrefix(String... prefixes) {
         String prefix = null;
         if (prefixes != null && prefixes.length > 0) {
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index 3f590c18c4d21..ea6e8f5bebee0 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -123,8 +123,7 @@ public static ScoreMode fromString(String scoreMode) {
     final ScoreMode scoreMode;
     final float maxBoost;
     private final Float minScore;
-
-    protected final CombineFunction combineFunction;
+    private final CombineFunction combineFunction;
 
     /**
      * Creates a FunctionScoreQuery without function.
@@ -192,6 +191,10 @@ public Float getMinScore() {
         return minScore;
     }
 
+    public CombineFunction getCombineFunction() {
+        return combineFunction;
+    }
+
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = super.rewrite(reader);
diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index b897b318b9dd3..d80d90cd5fb49 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -49,22 +49,24 @@
  */
 public abstract class AbstractScopedSettings extends AbstractComponent {
     public static final String ARCHIVED_SETTINGS_PREFIX = "archived.";
-    private Settings lastSettingsApplied = Settings.EMPTY;
+    private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
+    private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
+    private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$");
+
+    private final Settings settings;
     private final List<SettingUpdater<?>> settingUpdaters = new CopyOnWriteArrayList<>();
     private final Map<String, Setting<?>> complexMatchers;
     private final Map<String, Setting<?>> keySettings;
     private final Map<Setting<?>, SettingUpgrader<?>> settingUpgraders;
     private final Setting.Property scope;
-    private static final Pattern KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])*[-\\w]+$");
-    private static final Pattern GROUP_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+$");
-    private static final Pattern AFFIX_KEY_PATTERN = Pattern.compile("^(?:[-\\w]+[.])+[*](?:[.][-\\w]+)+$");
+    private Settings lastSettingsApplied;
 
     protected AbstractScopedSettings(
             final Settings settings,
             final Set<Setting<?>> settingsSet,
             final Set<SettingUpgrader<?>> settingUpgraders,
             final Setting.Property scope) {
-        super(settings);
+        this.settings = settings;
         this.lastSettingsApplied = Settings.EMPTY;
 
         this.settingUpgraders =
@@ -104,7 +106,7 @@ protected void validateSettingKey(Setting setting) {
     }
 
     protected AbstractScopedSettings(Settings nodeSettings, Settings scopeSettings, AbstractScopedSettings other) {
-        super(nodeSettings);
+        this.settings = nodeSettings;
         this.lastSettingsApplied = scopeSettings;
         this.scope = other.scope;
         complexMatchers = other.complexMatchers;
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index f387205dfa8cf..f78f645d902d5 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -295,6 +295,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     RemoteClusterService.SEARCH_REMOTE_NODE_ATTRIBUTE,
                     RemoteClusterService.ENABLE_REMOTE_CLUSTERS,
                     RemoteClusterService.SEARCH_ENABLE_REMOTE_CLUSTERS,
+                    RemoteClusterService.REMOTE_CLUSTER_PING_SCHEDULE,
                     TransportService.TRACE_LOG_EXCLUDE_SETTING,
                     TransportService.TRACE_LOG_INCLUDE_SETTING,
                     TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
index 1c67318e28286..fc68d58d5de0e 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
@@ -16,10 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.rest.RestRequest;
@@ -36,7 +36,7 @@
  * A class that allows to filter settings objects by simple regular expression patterns or full settings keys.
  * It's used for response filtering on the rest layer to for instance filter out sensitive information like access keys.
  */
-public final class SettingsFilter extends AbstractComponent {
+public final class SettingsFilter {
     /**
      * Can be used to specify settings filter that will be used to filter out matching settings in toXContent method
      */
@@ -45,8 +45,7 @@ public final class SettingsFilter extends AbstractComponent {
     private final Set<String> patterns;
     private final String patternString;
 
-    public SettingsFilter(Settings settings, Collection<String> patterns) {
-        super(settings);
+    public SettingsFilter(Collection<String> patterns) {
         for (String pattern : patterns) {
             if (isValidPattern(pattern) == false) {
                 throw new IllegalArgumentException("invalid pattern: " + pattern);
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 086346f470a50..6a78e81d7f3f4 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -146,7 +146,7 @@ public SettingsModule(
         }
         // by now we are fully configured, lets check node level settings for unregistered index settings
         clusterSettings.validate(settings, true);
-        this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern);
+        this.settingsFilter = new SettingsFilter(settingsFilterPattern);
      }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java
index ed04321ee83c5..7d25b75776ff8 100644
--- a/server/src/main/java/org/elasticsearch/common/time/DateUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/time/DateUtils.java
@@ -28,6 +28,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 public class DateUtils {
     public static DateTimeZone zoneIdToDateTimeZone(ZoneId zoneId) {
@@ -44,6 +45,7 @@ public static DateTimeZone zoneIdToDateTimeZone(ZoneId zoneId) {
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(DateFormatters.class));
     // pkg private for tests
     static final Map<String, String> DEPRECATED_SHORT_TIMEZONES;
+    public static final Set<String> DEPRECATED_SHORT_TZ_IDS;
     static {
         Map<String, String> tzs = new HashMap<>();
         tzs.put("EST", "-05:00"); // eastern time without daylight savings
@@ -52,6 +54,7 @@ public static DateTimeZone zoneIdToDateTimeZone(ZoneId zoneId) {
         tzs.put("ROC", "Asia/Taipei");
         tzs.put("Eire", "Europe/London");
         DEPRECATED_SHORT_TIMEZONES = Collections.unmodifiableMap(tzs);
+        DEPRECATED_SHORT_TZ_IDS = tzs.keySet();
     }
 
     public static ZoneId dateTimeZoneToZoneId(DateTimeZone timeZone) {
diff --git a/server/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java b/server/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
index be69cf5b95a06..6f2e28e97cbae 100644
--- a/server/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
+++ b/server/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
@@ -66,7 +66,6 @@ public void close() {
     }
 
     public PageCacheRecycler(Settings settings) {
-        super(settings);
         final Type type = TYPE_SETTING.get(settings);
         final long limit = LIMIT_HEAP_SETTING.get(settings).getBytes();
         final int availableProcessors = EsExecutors.numberOfProcessors(settings);
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index abc95810ba9a9..44367053406e3 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -179,6 +179,11 @@ public static ThreadFactory daemonThreadFactory(Settings settings, String namePr
         return daemonThreadFactory(threadName(settings, namePrefix));
     }
 
+    public static ThreadFactory daemonThreadFactory(String nodeName, String namePrefix) {
+        assert nodeName != null && false == nodeName.isEmpty();
+        return daemonThreadFactory(threadName(nodeName, namePrefix));
+    }
+
     public static ThreadFactory daemonThreadFactory(Settings settings, String ... names) {
         return daemonThreadFactory(threadName(settings, names));
     }
diff --git a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index 7256bb16747d8..91001d86b0f89 100644
--- a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -77,7 +77,7 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic
         final Collection<BiConsumer<DiscoveryNode,ClusterState>> joinValidators = new ArrayList<>();
         final Map<String, Supplier<UnicastHostsProvider>> hostProviders = new HashMap<>();
         hostProviders.put("settings", () -> new SettingsBasedHostsProvider(settings, transportService));
-        hostProviders.put("file", () -> new FileBasedUnicastHostsProvider(settings, configFile));
+        hostProviders.put("file", () -> new FileBasedUnicastHostsProvider(configFile));
         for (DiscoveryPlugin plugin : plugins) {
             plugin.getZenHostsProviders(transportService, networkService).entrySet().forEach(entry -> {
                 if (hostProviders.put(entry.getKey(), entry.getValue()) != null) {
diff --git a/server/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java b/server/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
index 5b7613587cd6f..ebc64fa3af1d1 100644
--- a/server/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
+++ b/server/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
@@ -72,7 +72,6 @@ public class DiscoverySettings extends AbstractComponent {
     private volatile boolean publishDiff;
 
     public DiscoverySettings(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
         clusterSettings.addSettingsUpdateConsumer(NO_MASTER_BLOCK_SETTING, this::setNoMasterBlock);
         clusterSettings.addSettingsUpdateConsumer(PUBLISH_DIFF_ENABLE_SETTING, this::setPublishDiff);
         clusterSettings.addSettingsUpdateConsumer(COMMIT_TIMEOUT_SETTING, this::setCommitTimeout);
diff --git a/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java b/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
index d7c37febb5dd2..75c69419723c3 100644
--- a/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
+++ b/server/src/main/java/org/elasticsearch/discovery/single/SingleNodeDiscovery.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -46,6 +47,7 @@
  */
 public class SingleNodeDiscovery extends AbstractLifecycleComponent implements Discovery {
 
+    private final ClusterName clusterName;
     protected final TransportService transportService;
     private final ClusterApplier clusterApplier;
     private volatile ClusterState clusterState;
@@ -53,6 +55,7 @@ public class SingleNodeDiscovery extends AbstractLifecycleComponent implements D
     public SingleNodeDiscovery(final Settings settings, final TransportService transportService,
                                final MasterService masterService, final ClusterApplier clusterApplier) {
         super(Objects.requireNonNull(settings));
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.transportService = Objects.requireNonNull(transportService);
         masterService.setClusterStateSupplier(() -> clusterState);
         this.clusterApplier = clusterApplier;
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
index ebce175e98118..8f2853904fa0f 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
@@ -98,7 +98,6 @@ public static int compare(MasterCandidate c1, MasterCandidate c2) {
     }
 
     public ElectMasterService(Settings settings) {
-        super(settings);
         this.minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings);
         logger.debug("using minimum_master_nodes [{}]", minimumMasterNodes);
     }
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
index 5d9b1687e4295..3d389fc814188 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/FaultDetection.java
@@ -66,7 +66,6 @@ public abstract class FaultDetection extends AbstractComponent implements Closea
     protected final int pingRetryCount;
 
     public FaultDetection(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterName clusterName) {
-        super(settings);
         this.threadPool = threadPool;
         this.transportService = transportService;
         this.clusterName = clusterName;
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java b/server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java
index 43caf418665cc..6692789595039 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProvider.java
@@ -24,7 +24,6 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 
 import java.io.IOException;
@@ -56,8 +55,7 @@ public class FileBasedUnicastHostsProvider extends AbstractComponent implements
     private final Path unicastHostsFilePath;
     private final Path legacyUnicastHostsFilePath;
 
-    public FileBasedUnicastHostsProvider(Settings settings, Path configFile) {
-        super(settings);
+    public FileBasedUnicastHostsProvider(Path configFile) {
         this.unicastHostsFilePath = configFile.resolve(UNICAST_HOSTS_FILE);
         this.legacyUnicastHostsFilePath = configFile.resolve("discovery-file").resolve(UNICAST_HOSTS_FILE);
     }
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java b/server/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
index fdfcd8ac29079..78247903359ad 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
@@ -66,9 +65,8 @@ public interface MembershipListener {
 
     private final MembershipListener listener;
 
-    public MembershipAction(Settings settings, TransportService transportService, MembershipListener listener,
+    public MembershipAction(TransportService transportService, MembershipListener listener,
                             Collection<BiConsumer<DiscoveryNode,ClusterState>> joinValidators) {
-        super(settings);
         this.transportService = transportService;
         this.listener = listener;
 
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/server/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 5cceba237e544..ecf52a6975369 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -65,9 +64,7 @@ public class NodeJoinController extends AbstractComponent {
     private ElectionContext electionContext = null;
 
 
-    public NodeJoinController(MasterService masterService, AllocationService allocationService, ElectMasterService electMaster,
-                              Settings settings) {
-        super(settings);
+    public NodeJoinController(MasterService masterService, AllocationService allocationService, ElectMasterService electMaster) {
         this.masterService = masterService;
         joinTaskExecutor = new JoinTaskExecutor(allocationService, electMaster, logger);
     }
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
index 5398b2a057ae4..580af49c0e007 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.discovery.AckClusterStatePublishResponseHandler;
@@ -96,12 +95,10 @@ public interface IncomingClusterStateListener {
     private final AtomicLong compatibleClusterStateDiffReceivedCount = new AtomicLong();
 
     public PublishClusterStateAction(
-            Settings settings,
             TransportService transportService,
             NamedWriteableRegistry namedWriteableRegistry,
             IncomingClusterStateListener incomingClusterStateListener,
             DiscoverySettings discoverySettings) {
-        super(settings);
         this.transportService = transportService;
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.incomingClusterStateListener = incomingClusterStateListener;
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/SettingsBasedHostsProvider.java b/server/src/main/java/org/elasticsearch/discovery/zen/SettingsBasedHostsProvider.java
index 6d6453c776e68..a11e255f88878 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/SettingsBasedHostsProvider.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/SettingsBasedHostsProvider.java
@@ -52,8 +52,6 @@ public class SettingsBasedHostsProvider extends AbstractComponent implements Uni
     private final int limitPortCounts;
 
     public SettingsBasedHostsProvider(Settings settings, TransportService transportService) {
-        super(settings);
-
         if (DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.exists(settings)) {
             configuredHosts = DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.get(settings);
             // we only limit to 1 address, makes no sense to ping 100 ports
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/server/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
index bba58de4f7e91..acb8ec478409c 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
@@ -123,7 +123,6 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
 
     public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
                           UnicastHostsProvider unicastHostsProvider, PingContextProvider contextProvider) {
-        super(settings);
         this.threadPool = threadPool;
         this.transportService = transportService;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index aadd43d8f5b81..6f51a6d20bbeb 100644
--- a/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/server/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -120,6 +120,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     private final NodesFaultDetection nodesFD;
     private final PublishClusterStateAction publishClusterState;
     private final MembershipAction membership;
+    private final ClusterName clusterName;
     private final ThreadPool threadPool;
 
     private final TimeValue pingTimeout;
@@ -171,7 +172,7 @@ public ZenDiscovery(Settings settings, ThreadPool threadPool, TransportService t
         this.maxPingsFromAnotherMaster = MAX_PINGS_FROM_ANOTHER_MASTER_SETTING.get(settings);
         this.sendLeaveRequest = SEND_LEAVE_REQUEST_SETTING.get(settings);
         this.threadPool = threadPool;
-        ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.committedState = new AtomicReference<>();
 
         this.masterElectionIgnoreNonMasters = MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING.get(settings);
@@ -209,15 +210,14 @@ public ZenDiscovery(Settings settings, ThreadPool threadPool, TransportService t
 
         this.publishClusterState =
                 new PublishClusterStateAction(
-                        settings,
                         transportService,
                         namedWriteableRegistry,
                         this,
                         discoverySettings);
-        this.membership = new MembershipAction(settings, transportService, new MembershipListener(), onJoinValidators);
+        this.membership = new MembershipAction(transportService, new MembershipListener(), onJoinValidators);
         this.joinThreadControl = new JoinThreadControl();
 
-        this.nodeJoinController = new NodeJoinController(masterService, allocationService, electMaster, settings);
+        this.nodeJoinController = new NodeJoinController(masterService, allocationService, electMaster);
         this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, electMaster, this::submitRejoin, logger);
 
         masterService.setClusterStateSupplier(this::clusterState);
diff --git a/server/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java b/server/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java
index 275311cf6a8f5..ba29a08987d3c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/BaseGatewayShardAllocator.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -42,11 +41,6 @@
  * the logic to determine to which nodes (if any) those shards are allocated.
  */
 public abstract class BaseGatewayShardAllocator extends AbstractComponent {
-
-    public BaseGatewayShardAllocator(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Allocate unassigned shards to nodes (if any) where valid copies of the shard already exist.
      * It is up to the individual implementations of {@link #makeAllocationDecision(ShardRouting, RoutingAllocation, Logger)}
diff --git a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index acfcadb2f51b5..d9eb5013e9c6c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -60,9 +59,8 @@ public class DanglingIndicesState extends AbstractComponent implements ClusterSt
     private final Map<Index, IndexMetaData> danglingIndices = ConcurrentCollections.newConcurrentMap();
 
     @Inject
-    public DanglingIndicesState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
+    public DanglingIndicesState(NodeEnvironment nodeEnv, MetaStateService metaStateService,
                                 LocalAllocateDangledIndices allocateDangledIndices, ClusterService clusterService) {
-        super(settings);
         this.nodeEnv = nodeEnv;
         this.metaStateService = metaStateService;
         this.allocateDangledIndices = allocateDangledIndices;
diff --git a/server/src/main/java/org/elasticsearch/gateway/Gateway.java b/server/src/main/java/org/elasticsearch/gateway/Gateway.java
index afd0964004292..1248d9c888c91 100644
--- a/server/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/server/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -49,7 +49,6 @@ public class Gateway extends AbstractComponent {
     public Gateway(Settings settings, ClusterService clusterService,
                    TransportNodesListGatewayMetaState listGatewayMetaState,
                    IndicesService indicesService) {
-        super(settings);
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.listGatewayMetaState = listGatewayMetaState;
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 4e7266a68320d..dce92b1dd5083 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
@@ -54,12 +53,11 @@ public class GatewayAllocator extends AbstractComponent {
         asyncFetchStore = ConcurrentCollections.newConcurrentMap();
 
     @Inject
-    public GatewayAllocator(Settings settings, ClusterService clusterService, RoutingService routingService,
+    public GatewayAllocator(ClusterService clusterService, RoutingService routingService,
                             TransportNodesListGatewayStartedShards startedAction, TransportNodesListShardStoreMetaData storeAction) {
-        super(settings);
         this.routingService = routingService;
-        this.primaryShardAllocator = new InternalPrimaryShardAllocator(settings, startedAction);
-        this.replicaShardAllocator = new InternalReplicaShardAllocator(settings, storeAction);
+        this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction);
+        this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction);
         clusterService.addStateApplier(event -> {
             boolean cleanCache = false;
             DiscoveryNode localNode = event.state().nodes().getLocalNode();
@@ -80,8 +78,7 @@ public GatewayAllocator(Settings settings, ClusterService clusterService, Routin
     }
 
     // for tests
-    protected GatewayAllocator(Settings settings) {
-        super(settings);
+    protected GatewayAllocator() {
         this.routingService = null;
         this.primaryShardAllocator = null;
         this.replicaShardAllocator = null;
@@ -157,8 +154,7 @@ class InternalPrimaryShardAllocator extends PrimaryShardAllocator {
 
         private final TransportNodesListGatewayStartedShards startedAction;
 
-        InternalPrimaryShardAllocator(Settings settings, TransportNodesListGatewayStartedShards startedAction) {
-            super(settings);
+        InternalPrimaryShardAllocator(TransportNodesListGatewayStartedShards startedAction) {
             this.startedAction = startedAction;
         }
 
@@ -182,8 +178,7 @@ class InternalReplicaShardAllocator extends ReplicaShardAllocator {
 
         private final TransportNodesListShardStoreMetaData storeAction;
 
-        InternalReplicaShardAllocator(Settings settings, TransportNodesListShardStoreMetaData storeAction) {
-            super(settings);
+        InternalReplicaShardAllocator(TransportNodesListShardStoreMetaData storeAction) {
             this.storeAction = storeAction;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index fd65d17b1d8cc..9bbb5af5bf028 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -69,7 +69,6 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA
 
     public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService,
                             MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException {
-        super(settings);
         this.nodeEnv = nodeEnv;
         this.metaStateService = metaStateService;
 
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
index e8e442ab64b84..e19e8367f1627 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -101,22 +101,22 @@ public GatewayService(Settings settings, AllocationService allocationService, Cl
         this.clusterService = clusterService;
         this.threadPool = threadPool;
         // allow to control a delay of when indices will get created
-        this.expectedNodes = EXPECTED_NODES_SETTING.get(this.settings);
-        this.expectedDataNodes = EXPECTED_DATA_NODES_SETTING.get(this.settings);
-        this.expectedMasterNodes = EXPECTED_MASTER_NODES_SETTING.get(this.settings);
+        this.expectedNodes = EXPECTED_NODES_SETTING.get(settings);
+        this.expectedDataNodes = EXPECTED_DATA_NODES_SETTING.get(settings);
+        this.expectedMasterNodes = EXPECTED_MASTER_NODES_SETTING.get(settings);
 
-        if (RECOVER_AFTER_TIME_SETTING.exists(this.settings)) {
-            recoverAfterTime = RECOVER_AFTER_TIME_SETTING.get(this.settings);
+        if (RECOVER_AFTER_TIME_SETTING.exists(settings)) {
+            recoverAfterTime = RECOVER_AFTER_TIME_SETTING.get(settings);
         } else if (expectedNodes >= 0 || expectedDataNodes >= 0 || expectedMasterNodes >= 0) {
             recoverAfterTime = DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET;
         } else {
             recoverAfterTime = null;
         }
-        this.recoverAfterNodes = RECOVER_AFTER_NODES_SETTING.get(this.settings);
-        this.recoverAfterDataNodes = RECOVER_AFTER_DATA_NODES_SETTING.get(this.settings);
+        this.recoverAfterNodes = RECOVER_AFTER_NODES_SETTING.get(settings);
+        this.recoverAfterDataNodes = RECOVER_AFTER_DATA_NODES_SETTING.get(settings);
         // default the recover after master nodes to the minimum master nodes in the discovery
-        if (RECOVER_AFTER_MASTER_NODES_SETTING.exists(this.settings)) {
-            recoverAfterMasterNodes = RECOVER_AFTER_MASTER_NODES_SETTING.get(this.settings);
+        if (RECOVER_AFTER_MASTER_NODES_SETTING.exists(settings)) {
+            recoverAfterMasterNodes = RECOVER_AFTER_MASTER_NODES_SETTING.get(settings);
         } else {
             // TODO: change me once the minimum_master_nodes is changed too
             recoverAfterMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", -1);
diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 28fbc045799f0..11184e7427240 100644
--- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
@@ -63,9 +62,8 @@ public class LocalAllocateDangledIndices extends AbstractComponent {
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
 
     @Inject
-    public LocalAllocateDangledIndices(Settings settings, TransportService transportService, ClusterService clusterService,
+    public LocalAllocateDangledIndices(TransportService transportService, ClusterService clusterService,
                                        AllocationService allocationService, MetaDataIndexUpgradeService metaDataIndexUpgradeService) {
-        super(settings);
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index 9377247488e7b..24f5fd63662d9 100644
--- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -42,8 +41,7 @@ public class MetaStateService extends AbstractComponent {
     private final NodeEnvironment nodeEnv;
     private final NamedXContentRegistry namedXContentRegistry;
 
-    public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) {
-        super(settings);
+    public MetaStateService(NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) {
         this.nodeEnv = nodeEnv;
         this.namedXContentRegistry = namedXContentRegistry;
     }
diff --git a/server/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/server/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
index d79f23be72bf7..79030336acc02 100644
--- a/server/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.gateway.AsyncShardFetch.FetchResult;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
@@ -63,11 +62,6 @@
  * copy that can immediately be promoted to primary, as this takes place in {@link RoutingNodes#failShard}.
  */
 public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
-
-    public PrimaryShardAllocator(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Is the allocator responsible for allocating the given {@link ShardRouting}?
      */
diff --git a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 777e8e31505d7..10bd6115b4c74 100644
--- a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.store.StoreFileMetaData;
@@ -55,11 +54,6 @@
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 
 public abstract class ReplicaShardAllocator extends BaseGatewayShardAllocator {
-
-    public ReplicaShardAllocator(Settings settings) {
-        super(settings);
-    }
-
     /**
      * Process existing recoveries of replicas and see if we need to cancel them if we find a better
      * match. Today, a better match is one that has full sync id match compared to not having one in
diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 24344c69a7ec7..f08c5b5131d77 100644
--- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -68,6 +68,7 @@ public class TransportNodesListGatewayStartedShards extends
         TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> {
 
     public static final String ACTION_NAME = "internal:gateway/local/started_shards";
+    private final Settings settings;
     private final NodeEnvironment nodeEnv;
     private final IndicesService indicesService;
     private final NamedXContentRegistry namedXContentRegistry;
@@ -82,6 +83,7 @@ public TransportNodesListGatewayStartedShards(Settings settings, ThreadPool thre
         super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STARTED,
               NodeGatewayStartedShards.class);
+        this.settings = settings;
         this.nodeEnv = env;
         this.indicesService = indicesService;
         this.namedXContentRegistry = namedXContentRegistry;
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
index d57e52de87a3f..d5801e099526f 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -187,8 +187,7 @@ public IndexService(
         this.indexStore = indexStore;
         indexFieldData.setListener(new FieldDataCacheListener(this));
         this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
-        this.warmer = new IndexWarmer(indexSettings.getSettings(), threadPool, indexFieldData,
-            bitsetFilterCache.createListener(threadPool));
+        this.warmer = new IndexWarmer(threadPool, indexFieldData, bitsetFilterCache.createListener(threadPool));
         this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
         this.engineFactory = Objects.requireNonNull(engineFactory);
         // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
index 213d1b5606ae1..ce06b68dcff23 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -22,7 +22,6 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -46,9 +45,8 @@ public final class IndexWarmer extends AbstractComponent {
 
     private final List<Listener> listeners;
 
-    IndexWarmer(Settings settings, ThreadPool threadPool, IndexFieldDataService indexFieldDataService,
+    IndexWarmer(ThreadPool threadPool, IndexFieldDataService indexFieldDataService,
                 Listener... listeners) {
-        super(settings);
         ArrayList<Listener> list = new ArrayList<>();
         final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
         list.add(new FieldDataWarmer(executor, indexFieldDataService));
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index aff51bcb3f9b5..cc1c199a601e6 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -2643,12 +2643,19 @@ final long lastRefreshedCheckpoint() {
         return lastRefreshedCheckpointListener.refreshedCheckpoint.get();
     }
 
+
+    private final Object refreshIfNeededMutex = new Object();
+
     /**
      * Refresh this engine **internally** iff the requesting seq_no is greater than the last refreshed checkpoint.
      */
     protected final void refreshIfNeeded(String source, long requestingSeqNo) {
         if (lastRefreshedCheckpoint() < requestingSeqNo) {
-            refresh(source, SearcherScope.INTERNAL);
+            synchronized (refreshIfNeededMutex) {
+                if (lastRefreshedCheckpoint() < requestingSeqNo) {
+                    refresh(source, SearcherScope.INTERNAL);
+                }
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
index 3b9f37b335948..ecb2a05c68438 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
@@ -96,14 +96,15 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
             }
         };
         this.mapperService = mapperService;
-        this.searchBatchSize = searchBatchSize;
+        final long requestingSize = (toSeqNo - fromSeqNo) == Long.MAX_VALUE ? Long.MAX_VALUE : (toSeqNo - fromSeqNo + 1L);
+        this.searchBatchSize = requestingSize < searchBatchSize ? Math.toIntExact(requestingSize) : searchBatchSize;
         this.fromSeqNo = fromSeqNo;
         this.toSeqNo = toSeqNo;
         this.lastSeenSeqNo = fromSeqNo - 1;
         this.requiredFullRange = requiredFullRange;
         this.indexSearcher = new IndexSearcher(Lucene.wrapAllDocsLive(engineSearcher.getDirectoryReader()));
         this.indexSearcher.setQueryCache(null);
-        this.parallelArray = new ParallelArray(searchBatchSize);
+        this.parallelArray = new ParallelArray(this.searchBatchSize);
         final TopDocs topDocs = searchOperations(null);
         this.totalHits = Math.toIntExact(topDocs.totalHits);
         this.scoreDocs = topDocs.scoreDocs;
diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java
index 22577362b9acc..23d1f8b4f77c2 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java
@@ -57,16 +57,16 @@ final class TranslogLeafReader extends LeafReader {
     private final Translog.Index operation;
     private static final FieldInfo FAKE_SOURCE_FIELD
         = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0, 0, false);
+        0, 0, 0, false);
     private static final FieldInfo FAKE_ROUTING_FIELD
         = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0, 0, false);
+        0, 0, 0, false);
     private static final FieldInfo FAKE_ID_FIELD
         = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0, 0, false);
+        0, 0, 0, false);
     private static final FieldInfo FAKE_UID_FIELD
         = new FieldInfo(UidFieldMapper.NAME, 4, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0, 0, false);
+        0, 0, 0, false);
     private final Version indexVersionCreated;
 
     TranslogLeafReader(Translog.Index operation, Version indexVersionCreated) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
index deb88d13fef4e..c63976387d8af 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
@@ -117,7 +117,8 @@ public static class Fields {
     public static class TypeParser implements Mapper.TypeParser {
 
         @Override
-        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+                throws MapperParsingException {
             CompletionFieldMapper.Builder builder = new CompletionFieldMapper.Builder(name);
             NamedAnalyzer indexAnalyzer = null;
             NamedAnalyzer searchAnalyzer = null;
@@ -368,7 +369,8 @@ public Builder(String name) {
          */
         public Builder maxInputLength(int maxInputLength) {
             if (maxInputLength <= 0) {
-                throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() + " must be > 0 but was [" + maxInputLength + "]");
+                throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName()
+                    + " must be > 0 but was [" + maxInputLength + "]");
             }
             this.maxInputLength = maxInputLength;
             return this;
@@ -400,13 +402,15 @@ public CompletionFieldMapper build(BuilderContext context) {
             completionFieldType.setContextMappings(contextMappings);
             completionFieldType.setPreservePositionIncrements(preservePositionIncrements);
             completionFieldType.setPreserveSep(preserveSeparators);
-            return new CompletionFieldMapper(name, this.fieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, maxInputLength);
+            return new CompletionFieldMapper(name, this.fieldType, context.indexSettings(),
+                multiFieldsBuilder.build(this, context), copyTo, maxInputLength);
         }
     }
 
     private int maxInputLength;
 
-    public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, int maxInputLength) {
+    public CompletionFieldMapper(String simpleName, MappedFieldType fieldType, Settings indexSettings,
+                                 MultiFields multiFields, CopyTo copyTo, int maxInputLength) {
         super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
         this.maxInputLength = maxInputLength;
     }
@@ -506,7 +510,8 @@ private boolean isExternalValueOfClass(ParseContext context, Class<?> clazz) {
      *  "STRING" - interpreted as the field value (input)
      *  "OBJECT" - { "input": STRING|ARRAY, "weight": STRING|INT, "contexts": ARRAY|OBJECT }
      */
-    private void parse(ParseContext parseContext, Token token, XContentParser parser, Map<String, CompletionInputMetaData> inputMap) throws IOException {
+    private void parse(ParseContext parseContext, Token token,
+                       XContentParser parser, Map<String, CompletionInputMetaData> inputMap) throws IOException {
         String currentFieldName = null;
         if (token == Token.VALUE_STRING) {
             inputMap.put(parser.text(), new CompletionInputMetaData(parser.text(), Collections.emptyMap(), 1));
@@ -518,7 +523,8 @@ private void parse(ParseContext parseContext, Token token, XContentParser parser
                 if (token == Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                     if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) {
-                        throw new IllegalArgumentException("unknown field name [" + currentFieldName + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES);
+                        throw new IllegalArgumentException("unknown field name [" + currentFieldName
+                            + "], must be one of " + ALLOWED_CONTENT_FIELD_NAMES);
                     }
                 } else if (currentFieldName != null) {
                     if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) {
@@ -529,7 +535,8 @@ private void parse(ParseContext parseContext, Token token, XContentParser parser
                                 if (token == Token.VALUE_STRING) {
                                     inputs.add(parser.text());
                                 } else {
-                                    throw new IllegalArgumentException("input array must have string values, but was [" + token.name() + "]");
+                                    throw new IllegalArgumentException("input array must have string values, but was ["
+                                        + token.name() + "]");
                                 }
                             }
                         } else {
@@ -552,8 +559,10 @@ private void parse(ParseContext parseContext, Token token, XContentParser parser
                         } else {
                             throw new IllegalArgumentException("weight must be a number or string, but was [" + token.name() + "]");
                         }
-                        if (weightValue.longValue() < 0 || weightValue.longValue() > Integer.MAX_VALUE) { // always parse a long to make sure we don't get overflow
-                            throw new IllegalArgumentException("weight must be in the interval [0..2147483647], but was [" + weightValue.longValue() + "]");
+                        // always parse a long to make sure we don't get overflow
+                        if (weightValue.longValue() < 0 || weightValue.longValue() > Integer.MAX_VALUE) {
+                            throw new IllegalArgumentException("weight must be in the interval [0..2147483647], but was ["
+                                + weightValue.longValue() + "]");
                         }
                         weight = weightValue.intValue();
                     } else if (Fields.CONTENT_FIELD_NAME_CONTEXTS.equals(currentFieldName)) {
@@ -587,7 +596,8 @@ private void parse(ParseContext parseContext, Token token, XContentParser parser
                 }
             }
         } else {
-            throw new ParsingException(parser.getTokenLocation(), "failed to parse [" + parser.currentName() + "]: expected text or object, but got " + token.name());
+            throw new ParsingException(parser.getTokenLocation(), "failed to parse [" + parser.currentName()
+                + "]: expected text or object, but got " + token.name());
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 2d3cd655d28ff..ec3635611a98e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -71,8 +71,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
             this.rootObjectMapper = builder.build(builderContext);
 
             final String type = rootObjectMapper.name();
-            DocumentMapper existingMapper = mapperService.documentMapper(type);
-            for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : mapperService.mapperRegistry.getMetadataMapperParsers().entrySet()) {
+            final DocumentMapper existingMapper = mapperService.documentMapper(type);
+            final Map<String, TypeParser> metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers();
+            for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : metadataMapperParsers.entrySet()) {
                 final String name = entry.getKey();
                 final MetadataFieldMapper existingMetadataMapper = existingMapper == null
                         ? null
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index ef85ee7127acd..f7913e895e772 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -107,7 +107,8 @@ private void validateType(SourceToParse source) {
         }
 
         if (Objects.equals(source.type(), docMapper.type()) == false) {
-            throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]");
+            throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type ["
+                + docMapper.type() + "]");
         }
     }
 
@@ -136,7 +137,8 @@ private static boolean isEmptyDoc(Mapping mapping, XContentParser parser) throws
                 // empty doc, we can handle it...
                 return true;
             } else if (token != XContentParser.Token.FIELD_NAME) {
-                throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist");
+                throw new MapperParsingException("Malformed content, after first object, either the type field"
+                    + " or the actual properties should exist");
             }
         }
         return false;
@@ -355,7 +357,8 @@ static void parseObjectOrNested(ParseContext context, ObjectMapper mapper) throw
 
         String currentFieldName = parser.currentName();
         if (token.isValue()) {
-            throw new MapperParsingException("object mapping for [" + mapper.name() + "] tried to parse field [" + currentFieldName + "] as object, but found a concrete value");
+            throw new MapperParsingException("object mapping for [" + mapper.name() + "] tried to parse field [" + currentFieldName
+                + "] as object, but found a concrete value");
         }
 
         ObjectMapper.Nested nested = mapper.nested();
@@ -385,7 +388,8 @@ static void parseObjectOrNested(ParseContext context, ObjectMapper mapper) throw
         }
     }
 
-    private static void innerParseObject(ParseContext context, ObjectMapper mapper, XContentParser parser, String currentFieldName, XContentParser.Token token) throws IOException {
+    private static void innerParseObject(ParseContext context, ObjectMapper mapper, XContentParser parser,
+                                         String currentFieldName, XContentParser.Token token) throws IOException {
         while (token != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.START_OBJECT) {
                 parseObject(context, mapper, currentFieldName);
@@ -394,12 +398,14 @@ private static void innerParseObject(ParseContext context, ObjectMapper mapper,
             } else if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
                 if (MapperService.isMetadataField(context.path().pathAsText(currentFieldName))) {
-                    throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside a document. Use the index API request parameters.");
+                    throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside"
+                        + " a document. Use the index API request parameters.");
                 }
             } else if (token == XContentParser.Token.VALUE_NULL) {
                 parseNullValue(context, mapper, currentFieldName);
             } else if (token == null) {
-                throw new MapperParsingException("object mapping for [" + mapper.name() + "] tried to parse field [" + currentFieldName + "] as object, but got EOF, has a concrete value been provided to it?");
+                throw new MapperParsingException("object mapping for [" + mapper.name() + "] tried to parse field [" + currentFieldName
+                    + "] as object, but got EOF, has a concrete value been provided to it?");
             } else if (token.isValue()) {
                 parseValue(context, mapper, currentFieldName, token);
             }
@@ -575,7 +581,8 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
         }
     }
 
-    private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapper, String lastFieldName, String arrayFieldName) throws IOException {
+    private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapper,
+                                             String lastFieldName, String arrayFieldName) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
@@ -588,16 +595,19 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp
             } else if (token == XContentParser.Token.VALUE_NULL) {
                 parseNullValue(context, mapper, lastFieldName);
             } else if (token == null) {
-                throw new MapperParsingException("object mapping for [" + mapper.name() + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?");
+                throw new MapperParsingException("object mapping for [" + mapper.name() + "] with array for [" + arrayFieldName
+                    + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?");
             } else {
                 parseValue(context, mapper, lastFieldName, token);
             }
         }
     }
 
-    private static void parseValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
+    private static void parseValue(final ParseContext context, ObjectMapper parentMapper,
+                                   String currentFieldName, XContentParser.Token token) throws IOException {
         if (currentFieldName == null) {
-            throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]");
+            throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with"
+                + " no field associated with it, current value [" + context.parser().textOrNull() + "]");
         }
 
         final String[] paths = splitAndValidatePath(currentFieldName);
@@ -626,7 +636,8 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp
         }
     }
 
-    private static Mapper.Builder<?,?> createBuilderFromFieldType(final ParseContext context, MappedFieldType fieldType, String currentFieldName) {
+    private static Mapper.Builder<?,?> createBuilderFromFieldType(final ParseContext context,
+                                                                  MappedFieldType fieldType, String currentFieldName) {
         Mapper.Builder builder = null;
         if (fieldType instanceof TextFieldType) {
             builder = context.root().findTemplateBuilder(context, currentFieldName, "text", XContentFieldType.STRING);
@@ -688,7 +699,9 @@ private static Mapper.Builder<?,?> createBuilderFromFieldType(final ParseContext
         return builder;
     }
 
-    private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseContext context, XContentParser.Token token, String currentFieldName) throws IOException {
+    private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseContext context,
+                                                                     XContentParser.Token token,
+                                                                     String currentFieldName) throws IOException {
         if (token == XContentParser.Token.VALUE_STRING) {
             String text = context.parser().text();
 
@@ -788,10 +801,12 @@ private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseCont
             }
         }
         // TODO how do we identify dynamically that its a binary value?
-        throw new IllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]");
+        throw new IllegalStateException("Can't handle serializing a dynamic type with content token [" + token + "] and field name ["
+            + currentFieldName + "]");
     }
 
-    private static void parseDynamicValue(final ParseContext context, ObjectMapper parentMapper, String currentFieldName, XContentParser.Token token) throws IOException {
+    private static void parseDynamicValue(final ParseContext context, ObjectMapper parentMapper,
+                                          String currentFieldName, XContentParser.Token token) throws IOException {
         ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
         if (dynamic == ObjectMapper.Dynamic.STRICT) {
             throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName);
@@ -902,8 +917,8 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
                             context.path());
                         mapper = (ObjectMapper) builder.build(builderContext);
                         if (mapper.nested() != ObjectMapper.Nested.NO) {
-                            throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i])
-                                    + "]) through `copy_to` or dots in field names");
+                            throw new MapperParsingException("It is forbidden to create dynamic nested objects (["
+                                + context.path().pathAsText(paths[i]) + "]) through `copy_to` or dots in field names");
                         }
                         context.addDynamicMapper(mapper);
                         break;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
index 97b973934d016..eeaefebfe4ce9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
@@ -228,7 +228,8 @@ public static DynamicTemplate parse(String name, Map<String, Object> conf,
                 try {
                     matchType.matches(regex, "");
                 } catch (IllegalArgumentException e) {
-                    throw new IllegalArgumentException("Pattern [" + regex + "] of type [" + matchType + "] is invalid. Cannot create dynamic template [" + name + "].", e);
+                    throw new IllegalArgumentException("Pattern [" + regex + "] of type [" + matchType
+                        + "] is invalid. Cannot create dynamic template [" + name + "].", e);
                 }
             }
         }
@@ -320,14 +321,16 @@ public Map<String, Object> mappingForName(String name, String dynamicType) {
     private Map<String, Object> processMap(Map<String, Object> map, String name, String dynamicType) {
         Map<String, Object> processedMap = new HashMap<>();
         for (Map.Entry<String, Object> entry : map.entrySet()) {
-            String key = entry.getKey().replace("{name}", name).replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType);
+            String key = entry.getKey().replace("{name}", name).replace("{dynamic_type}", dynamicType)
+                .replace("{dynamicType}", dynamicType);
             Object value = entry.getValue();
             if (value instanceof Map) {
                 value = processMap((Map<String, Object>) value, name, dynamicType);
             } else if (value instanceof List) {
                 value = processList((List) value, name, dynamicType);
             } else if (value instanceof String) {
-                value = value.toString().replace("{name}", name).replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType);
+                value = value.toString().replace("{name}", name).replace("{dynamic_type}", dynamicType)
+                    .replace("{dynamicType}", dynamicType);
             }
             processedMap.put(key, value);
         }
@@ -342,7 +345,9 @@ private List processList(List list, String name, String dynamicType) {
             } else if (value instanceof List) {
                 value = processList((List) value, name, dynamicType);
             } else if (value instanceof String) {
-                value = value.toString().replace("{name}", name).replace("{dynamic_type}", dynamicType).replace("{dynamicType}", dynamicType);
+                value = value.toString().replace("{name}", name)
+                    .replace("{dynamic_type}", dynamicType)
+                    .replace("{dynamicType}", dynamicType);
             }
             processedList.add(value);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 0a5ecc9e7a82b..24f07c07d34d5 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -92,7 +92,8 @@ public T index(boolean index) {
                         // can happen when an existing type on the same index has disabled indexing
                         // since we inherit the default field type from the first mapper that is
                         // created on an index
-                        throw new IllegalArgumentException("mapper [" + name + "] has different [index] values from other types of the same index");
+                        throw new IllegalArgumentException("mapper [" + name + "] has different [index] values from other types"
+                            + " of the same index");
                     }
                     fieldType.setIndexOptions(options);
                 }
@@ -248,7 +249,8 @@ protected void setupFieldType(BuilderContext context) {
     protected MultiFields multiFields;
     protected CopyTo copyTo;
 
-    protected FieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+    protected FieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+                          Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName);
         assert indexSettings != null;
         this.indexCreatedVersion = Version.indexCreated(indexSettings);
@@ -349,7 +351,8 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
             if (mergeWith instanceof FieldMapper) {
                 mergedType = ((FieldMapper) mergeWith).contentType();
             }
-            throw new IllegalArgumentException("mapper [" + fieldType().name() + "] of different type, current_type [" + contentType() + "], merged_type [" + mergedType + "]");
+            throw new IllegalArgumentException("mapper [" + fieldType().name() + "] of different type, current_type [" + contentType()
+                + "], merged_type [" + mergedType + "]");
         }
         FieldMapper fieldMergeWith = (FieldMapper) mergeWith;
         multiFields = multiFields.merge(fieldMergeWith.multiFields);
@@ -438,12 +441,13 @@ protected final void doXContentAnalyzers(XContentBuilder builder, boolean includ
             }
         } else {
             boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default");
-            boolean hasDifferentSearchAnalyzer = fieldType().searchAnalyzer().name().equals(fieldType().indexAnalyzer().name()) == false;
-            boolean hasDifferentSearchQuoteAnalyzer = fieldType().searchAnalyzer().name().equals(fieldType().searchQuoteAnalyzer().name()) == false;
+            final String searchAnalyzerName = fieldType().searchAnalyzer().name();
+            boolean hasDifferentSearchAnalyzer = searchAnalyzerName.equals(fieldType().indexAnalyzer().name()) == false;
+            boolean hasDifferentSearchQuoteAnalyzer = searchAnalyzerName.equals(fieldType().searchQuoteAnalyzer().name()) == false;
             if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) {
                 builder.field("analyzer", fieldType().indexAnalyzer().name());
                 if (includeDefaults || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) {
-                    builder.field("search_analyzer", fieldType().searchAnalyzer().name());
+                    builder.field("search_analyzer", searchAnalyzerName);
                     if (includeDefaults || hasDifferentSearchQuoteAnalyzer) {
                         builder.field("search_quote_analyzer", fieldType().searchQuoteAnalyzer().name());
                     }
@@ -545,7 +549,8 @@ private MultiFields(ImmutableOpenMap<String, FieldMapper> mappers) {
         }
 
         public void parse(FieldMapper mainField, ParseContext context) throws IOException {
-            // TODO: multi fields are really just copy fields, we just need to expose "sub fields" or something that can be part of the mappings
+            // TODO: multi fields are really just copy fields, we just need to expose "sub fields" or something that can be part
+            // of the mappings
             if (mappers.isEmpty()) {
                 return;
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
index 884470214ba8c..60cc563504249 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
@@ -103,7 +103,8 @@ public FieldNamesFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                      ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
 
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
@@ -170,7 +171,8 @@ public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts
             if (strict) {
                 FieldNamesFieldType other = (FieldNamesFieldType)fieldType;
                 if (isEnabled() != other.isEnabled()) {
-                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [enabled] across all types.");
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [enabled]"
+                        + " across all types.");
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index 48c68b8f3fee7..888bba664de0d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -210,7 +210,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString()));
                     iterator.remove();
                 } else if (Names.TREE_PRESISION.equals(fieldName)) {
-                    builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
+                    builder.fieldType().setPrecisionInMeters(DistanceUnit.parse(fieldNode.toString(),
+                        DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
                     iterator.remove();
                 } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) {
                     builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString()));
@@ -222,13 +223,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                     builder.fieldType().setStrategyName(fieldNode.toString());
                     iterator.remove();
                 } else if (IGNORE_MALFORMED.equals(fieldName)) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed",
+                        fieldNode, parserContext));
                     iterator.remove();
                 } else if (Names.COERCE.equals(fieldName)) {
                     builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
                     iterator.remove();
                 } else if (GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName().equals(fieldName)) {
-                    builder.ignoreZValue(TypeParsers.nodeBooleanValue(fieldName, GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(),
+                    builder.ignoreZValue(TypeParsers.nodeBooleanValue(fieldName,
+                        GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(),
                         fieldNode, parserContext));
                     iterator.remove();
                 } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)) {
@@ -314,11 +317,14 @@ public void freeze() {
             // must be by the time freeze is called.
             SpatialPrefixTree prefixTree;
             if ("geohash".equals(tree)) {
-                prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true));
+                prefixTree = new GeohashPrefixTree(ShapeBuilder.SPATIAL_CONTEXT,
+                    getLevels(treeLevels, precisionInMeters, Defaults.GEOHASH_LEVELS, true));
             } else if ("legacyquadtree".equals(tree)) {
-                prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false));
+                prefixTree = new QuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT,
+                    getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false));
             } else if ("quadtree".equals(tree)) {
-                prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT, getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false));
+                prefixTree = new PackedQuadPrefixTree(ShapeBuilder.SPATIAL_CONTEXT,
+                    getLevels(treeLevels, precisionInMeters, Defaults.QUADTREE_LEVELS, false));
             } else {
                 throw new IllegalArgumentException("Unknown prefix tree type [" + tree + "]");
             }
@@ -361,10 +367,12 @@ public void checkCompatibility(MappedFieldType fieldType, List<String> conflicts
 
             if (strict) {
                 if (orientation() != other.orientation()) {
-                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [orientation] across all types.");
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                        + " [orientation] across all types.");
                 }
                 if (distanceErrorPct() != other.distanceErrorPct()) {
-                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [distance_error_pct] across all types.");
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                        + " [distance_error_pct] across all types.");
                 }
             }
         }
@@ -512,8 +520,9 @@ public void parse(ParseContext context) throws IOException {
                     }
                     return;
                 } else if (shape instanceof Point == false) {
-                    throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " +
-                        ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + " was found");
+                    throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a "
+                        + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass())
+                        + " was found");
                 }
             }
             indexShape(context, shape);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
index ac8ff76018003..a4114759772fe 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -86,7 +86,8 @@ public static class Defaults {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
+                                                 ParserContext parserContext) throws MapperParsingException {
             throw new MapperParsingException(NAME + " is not configurable");
         }
 
@@ -171,7 +172,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
                 @Override
                 public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                         CircuitBreakerService breakerService, MapperService mapperService) {
-                    final IndexFieldData<?> fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache, breakerService, mapperService);
+                    final IndexFieldData<?> fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache,
+                        breakerService, mapperService);
                     if (indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_beta1)) {
                         // ids were indexed as utf-8
                         return fieldData;
@@ -200,7 +202,8 @@ public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
 
                         @Override
                         public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
-                            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+                            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
+                                sortMode, nested);
                             return new SortField(getFieldName(), source, reverse);
                         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
index 9ab899368c307..a9ff2d59dc4e3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
@@ -80,7 +80,8 @@ public IndexFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                      ParserContext parserContext) throws MapperParsingException {
             if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha3)) {
                 throw new MapperParsingException(NAME + " is not configurable");
             }
@@ -135,7 +136,8 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
             if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) {
                 return Queries.newMatchAllQuery();
             } else {
-                return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.index().getName() + " vs. " + value);
+                return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.index().getName()
+                    + " vs. " + value);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 6ba0b1be07f84..4d3a498a9ba01 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -139,9 +139,11 @@ public int hashCode() {
     /** Checks this type is the same type as other. Adds a conflict if they are different. */
     private void checkTypeName(MappedFieldType other) {
         if (typeName().equals(other.typeName()) == false) {
-            throw new IllegalArgumentException("mapper [" + name + "] cannot be changed from type [" + typeName() + "] to [" + other.typeName() + "]");
+            throw new IllegalArgumentException("mapper [" + name + "] cannot be changed from type [" + typeName()
+                + "] to [" + other.typeName() + "]");
         } else if (getClass() != other.getClass()) {
-            throw new IllegalStateException("Type names equal for class " + getClass().getSimpleName() + " and " + other.getClass().getSimpleName());
+            throw new IllegalStateException("Type names equal for class " + getClass().getSimpleName() + " and "
+                + other.getClass().getSimpleName());
         }
     }
 
@@ -198,22 +200,28 @@ public void checkCompatibility(MappedFieldType other, List<String> conflicts, bo
 
         if (strict) {
             if (omitNorms() != other.omitNorms()) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [omit_norms] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [omit_norms] across all types.");
             }
             if (boost() != other.boost()) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [boost] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [boost] across all types.");
             }
             if (Objects.equals(searchAnalyzer(), other.searchAnalyzer()) == false) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_analyzer] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [search_analyzer] across all types.");
             }
             if (Objects.equals(searchQuoteAnalyzer(), other.searchQuoteAnalyzer()) == false) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [search_quote_analyzer] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [search_quote_analyzer] across all types.");
             }
             if (Objects.equals(nullValue(), other.nullValue()) == false) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [null_value] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [null_value] across all types.");
             }
             if (eagerGlobalOrdinals() != other.eagerGlobalOrdinals()) {
-                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [eager_global_ordinals] across all types.");
+                conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update"
+                    + " [eager_global_ordinals] across all types.");
             }
         }
     }
@@ -360,21 +368,26 @@ public Query rangeQuery(
     }
 
     public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-        throw new IllegalArgumentException("Can only use fuzzy queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
+        throw new IllegalArgumentException("Can only use fuzzy queries on keyword and text fields - not on [" + name
+            + "] which is of type [" + typeName() + "]");
     }
 
     public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use prefix queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
+        throw new QueryShardException(context, "Can only use prefix queries on keyword and text fields - not on [" + name
+            + "] which is of type [" + typeName() + "]");
     }
 
     public Query wildcardQuery(String value,
                                @Nullable MultiTermQuery.RewriteMethod method,
                                QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use wildcard queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
+        throw new QueryShardException(context, "Can only use wildcard queries on keyword and text fields - not on [" + name
+            + "] which is of type [" + typeName() + "]");
     }
 
-    public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use regexp queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
+    public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method,
+                             QueryShardContext context) {
+        throw new QueryShardException(context, "Can only use regexp queries on keyword and text fields - not on [" + name
+            + "] which is of type [" + typeName() + "]");
     }
 
     public Query nullValueQuery() {
@@ -435,7 +448,7 @@ protected final void failIfNoDocValues() {
     }
 
     protected final void failIfNotIndexed() {
-        if (indexOptions() == IndexOptions.NONE && pointDimensionCount() == 0) {
+        if (indexOptions() == IndexOptions.NONE && pointDataDimensionCount() == 0) {
             // we throw an IAE rather than an ISE so that it translates to a 4xx code rather than 5xx code on the http layer
             throw new IllegalArgumentException("Cannot search on field [" + name() + "] since it is not indexed.");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index c8ecd0746f15a..fa3436655521a 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -158,7 +158,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
 
         if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_0_0_rc1)) {
             if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings())) {
-                DEPRECATION_LOGGER.deprecated("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " is deprecated since indices may not have more than one type anymore.");
+                DEPRECATION_LOGGER.deprecated("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey()
+                    + " is deprecated since indices may not have more than one type anymore.");
             }
             this.dynamic = INDEX_MAPPER_DYNAMIC_DEFAULT;
         } else {
@@ -196,7 +197,9 @@ public Iterable<DocumentMapper> docMappers(final boolean includingDefaultMapping
             if (includingDefaultMapping) {
                 documentMappers = mappers.values();
             } else {
-                documentMappers = mappers.values().stream().filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type())).collect(Collectors.toList());
+                documentMappers = mappers.values().stream()
+                    .filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type()))
+                    .collect(Collectors.toList());
             }
             return Collections.unmodifiableCollection(documentMappers).iterator();
         };
@@ -228,7 +231,8 @@ public static Map<String, Object> parseMapping(NamedXContentRegistry xContentReg
      * Update mapping by only merging the metadata that is different between received and stored entries
      */
     public boolean updateMapping(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) throws IOException {
-        assert newIndexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index() + " but was " + newIndexMetaData.getIndex();
+        assert newIndexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index()
+            + " but was " + newIndexMetaData.getIndex();
         // go over and add the relevant mappings (or update them)
         final Set<String> existingMappers = new HashSet<>(mappers.keySet());
         final Map<String, DocumentMapper> updatedEntries;
@@ -254,15 +258,16 @@ public boolean updateMapping(final IndexMetaData currentIndexMetaData, final Ind
             } else if (logger.isTraceEnabled()) {
                 logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, incomingMappingSource.string());
             } else {
-                logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op, mappingType);
+                logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)",
+                    index(), op, mappingType);
             }
 
             // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same
             // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the
             // merge version of it, which it does when refreshing the mappings), and warn log it.
             if (documentMapper(mappingType).mappingSource().equals(incomingMappingSource) == false) {
-                logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(), mappingType,
-                    incomingMappingSource, documentMapper(mappingType).mappingSource());
+                logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}",
+                    index(), mappingType, incomingMappingSource, documentMapper(mappingType).mappingSource());
 
                 requireRefresh = true;
             }
@@ -314,7 +319,8 @@ public void merge(Map<String, Map<String, Object>> mappings, MergeReason reason,
         Map<String, CompressedXContent> mappingSourcesCompressed = new LinkedHashMap<>(mappings.size());
         for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
             try {
-                mappingSourcesCompressed.put(entry.getKey(), new CompressedXContent(Strings.toString(XContentFactory.jsonBuilder().map(entry.getValue()))));
+                mappingSourcesCompressed.put(entry.getKey(), new CompressedXContent(Strings.toString(
+                    XContentFactory.jsonBuilder().map(entry.getValue()))));
             } catch (Exception e) {
                 throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
             }
@@ -331,8 +337,8 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge
         return internalMerge(Collections.singletonMap(type, mappingSource), reason, updateAllTypes).get(type);
     }
 
-    private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes,
-                                                                   boolean onlyUpdateIfNeeded) {
+    private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData indexMetaData, MergeReason reason,
+                                                                   boolean updateAllTypes, boolean onlyUpdateIfNeeded) {
         Map<String, CompressedXContent> map = new LinkedHashMap<>();
         for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
             MappingMetaData mappingMetaData = cursor.value;
@@ -348,7 +354,8 @@ private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData ind
         return internalMerge(map, reason, updateAllTypes);
     }
 
-    private synchronized Map<String, DocumentMapper> internalMerge(Map<String, CompressedXContent> mappings, MergeReason reason, boolean updateAllTypes) {
+    private synchronized Map<String, DocumentMapper> internalMerge(Map<String, CompressedXContent> mappings,
+                                                                   MergeReason reason, boolean updateAllTypes) {
         DocumentMapper defaultMapper = null;
         String defaultMappingSource = null;
 
@@ -404,10 +411,12 @@ static void validateTypeName(String type) {
             throw new InvalidTypeNameException("mapping type name is empty");
         }
         if (type.length() > 255) {
-            throw new InvalidTypeNameException("mapping type name [" + type + "] is too long; limit is length 255 but was [" + type.length() + "]");
+            throw new InvalidTypeNameException("mapping type name [" + type + "] is too long; limit is length 255 but was ["
+                + type.length() + "]");
         }
         if (type.charAt(0) == '_' && SINGLE_MAPPING_NAME.equals(type) == false) {
-            throw new InvalidTypeNameException("mapping type name [" + type + "] can't start with '_' unless it is called [" + SINGLE_MAPPING_NAME + "]");
+            throw new InvalidTypeNameException("mapping type name [" + type + "] can't start with '_' unless it is called ["
+                + SINGLE_MAPPING_NAME + "]");
         }
         if (type.contains("#")) {
             throw new InvalidTypeNameException("mapping type name [" + type + "] should not include '#' in it");
@@ -420,8 +429,10 @@ static void validateTypeName(String type) {
         }
     }
 
-    private synchronized Map<String, DocumentMapper> internalMerge(@Nullable DocumentMapper defaultMapper, @Nullable String defaultMappingSource,
-                                                                   List<DocumentMapper> documentMappers, MergeReason reason, boolean updateAllTypes) {
+    private synchronized Map<String, DocumentMapper> internalMerge(@Nullable DocumentMapper defaultMapper,
+                                                                   @Nullable String defaultMappingSource,
+                                                                   List<DocumentMapper> documentMappers,
+                                                                   MergeReason reason, boolean updateAllTypes) {
         boolean hasNested = this.hasNested;
         boolean allEnabled = this.allEnabled;
         Map<String, ObjectMapper> fullPathObjectMappers = this.fullPathObjectMappers;
@@ -448,7 +459,8 @@ private synchronized Map<String, DocumentMapper> internalMerge(@Nullable Documen
             actualTypes.remove(DEFAULT_MAPPING);
             if (actualTypes.size() > 1) {
                 throw new IllegalArgumentException(
-                    "Rejecting mapping update to [" + index().getName() + "] as the final mapping would have more than 1 type: " + actualTypes);
+                    "Rejecting mapping update to [" + index().getName() + "] as the final mapping would have more than 1 type: "
+                        + actualTypes);
             }
         }
 
@@ -508,7 +520,8 @@ private synchronized Map<String, DocumentMapper> internalMerge(@Nullable Documen
                 // deserializing cluster state that was sent by the master node,
                 // this check will be skipped.
                 // Also, don't take metadata mappers into account for the field limit check
-                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() - metadataMappers.length + fieldAliasMappers.size() );
+                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size() - metadataMappers.length
+                    + fieldAliasMappers.size() );
             }
 
             if (oldMapper == null && newMapper.parentFieldMapper().active()) {
@@ -618,14 +631,16 @@ private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMapp
             }
         }
         if (actualNestedFields > allowedNestedFields) {
-            throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName() + "] has been exceeded");
+            throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName()
+                + "] has been exceeded");
         }
     }
 
     private void checkTotalFieldsLimit(long totalMappers) {
         long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
         if (allowedTotalFields < totalMappers) {
-            throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() + "] has been exceeded");
+            throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName()
+                + "] has been exceeded");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
index 0ca2b2ea94447..7f59d4e623871 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java
@@ -48,7 +48,8 @@ public final class Mapping implements ToXContentFragment {
     final Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap;
     final Map<String, Object> meta;
 
-    public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, MetadataFieldMapper[] metadataMappers, Map<String, Object> meta) {
+    public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper,
+                   MetadataFieldMapper[] metadataMappers, Map<String, Object> meta) {
         this.indexCreated = indexCreated;
         this.metadataMappers = metadataMappers;
         Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap = new HashMap<>();
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
index 4e456cdc994ff..5b26b1a37bf7c 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
@@ -33,7 +33,8 @@ public abstract class MetadataFieldMapper extends FieldMapper {
     public interface TypeParser extends Mapper.TypeParser {
 
         @Override
-        MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;
+        MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                               ParserContext parserContext) throws MapperParsingException;
 
         /**
          * Get the default {@link MetadataFieldMapper} to use, if nothing had to be parsed.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index 4fa8623183931..13871fcb8927e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -177,7 +177,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
             return builder;
         }
 
-        protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, Object fieldNode, ParserContext parserContext, ObjectMapper.Builder builder) {
+        protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, Object fieldNode, ParserContext parserContext,
+                                                                     ObjectMapper.Builder builder) {
             if (fieldName.equals("dynamic")) {
                 String value = fieldNode.toString();
                 if (value.equalsIgnoreCase("strict")) {
@@ -219,7 +220,8 @@ protected static void parseNested(String name, Map<String, Object> node, ObjectM
                 } else if (type.equals(NESTED_CONTENT_TYPE)) {
                     nested = true;
                 } else {
-                    throw new MapperParsingException("Trying to parse an object but has a different type [" + type + "] for [" + name + "]");
+                    throw new MapperParsingException("Trying to parse an object but has a different type [" + type
+                        + "] for [" + name + "]");
                 }
             }
             fieldNode = node.get("include_in_parent");
@@ -447,7 +449,8 @@ public boolean parentObjectMapperAreNested(MapperService mapperService) {
     @Override
     public ObjectMapper merge(Mapper mergeWith, boolean updateAllTypes) {
         if (!(mergeWith instanceof ObjectMapper)) {
-            throw new IllegalArgumentException("Can't merge a non object mapping [" + mergeWith.name() + "] with an object mapping [" + name() + "]");
+            throw new IllegalArgumentException("Can't merge a non object mapping [" + mergeWith.name()
+                + "] with an object mapping [" + name() + "]");
         }
         ObjectMapper mergeWithObject = (ObjectMapper) mergeWith;
         ObjectMapper merged = clone();
@@ -537,7 +540,8 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
             if (nested.isIncludeInRoot()) {
                 builder.field("include_in_root", true);
             }
-        } else if (mappers.isEmpty() && custom == null) { // only write the object content type if there are no properties, otherwise, it is automatically detected
+        } else if (mappers.isEmpty() && custom == null) {
+            // only write the object content type if there are no properties, otherwise, it is automatically detected
             builder.field("type", CONTENT_TYPE);
         }
         if (dynamic != null) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
index 8cd049f1f5a48..4056c9556c45b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
@@ -106,14 +106,16 @@ public ParentFieldMapper build(BuilderContext context) {
             }
             name = joinField(parentType);
             setupFieldType(context);
-            return new ParentFieldMapper(createParentJoinFieldMapper(documentType, context), fieldType, parentType, context.indexSettings());
+            return new ParentFieldMapper(createParentJoinFieldMapper(documentType, context), fieldType,
+                parentType, context.indexSettings());
         }
     }
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         private static final ParseField FIELDDATA = new ParseField("fielddata").withAllDeprecated("eager_global_ordinals");
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
+                                                 ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.type());
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -141,7 +143,8 @@ public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node,
         public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
             final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
             final String typeName = context.type();
-            KeywordFieldMapper parentJoinField = createParentJoinFieldMapper(typeName, new BuilderContext(indexSettings, new ContentPath(0)));
+            KeywordFieldMapper parentJoinField = createParentJoinFieldMapper(typeName,
+                new BuilderContext(indexSettings, new ContentPath(0)));
             MappedFieldType childJoinFieldType = new ParentFieldType(Defaults.FIELD_TYPE, typeName);
             childJoinFieldType.setName(ParentFieldMapper.NAME);
             return new ParentFieldMapper(parentJoinField, childJoinFieldType, null, indexSettings);
@@ -218,7 +221,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
     // the parent field mapper in the child type pointing to this type determines the field data settings for this join field
     private final KeywordFieldMapper parentJoinField;
 
-    private ParentFieldMapper(KeywordFieldMapper parentJoinField, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) {
+    private ParentFieldMapper(KeywordFieldMapper parentJoinField, MappedFieldType childJoinFieldType,
+                              String parentType, Settings indexSettings) {
         super(NAME, childJoinFieldType, Defaults.FIELD_TYPE, indexSettings);
         this.parentType = parentType;
         this.parentJoinField = parentJoinField;
@@ -269,7 +273,8 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
                     // we did not add it in the parsing phase, add it now
                     fields.add(new SortedDocValuesField(fieldType.name(), new BytesRef(parentId)));
                 } else if (parentId != null && !parsedParentId.equals(Uid.createUid(parentType, parentId))) {
-                    throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]");
+                    throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id()
+                        + "], while external value is [" + parentId + "]");
                 }
             }
         }
@@ -310,7 +315,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
         ParentFieldMapper fieldMergeWith = (ParentFieldMapper) mergeWith;
         if (fieldMergeWith.parentType != null && Objects.equals(parentType, fieldMergeWith.parentType) == false) {
-            throw new IllegalArgumentException("The _parent field's type option can't be changed: [" + parentType + "]->[" + fieldMergeWith.parentType + "]");
+            throw new IllegalArgumentException("The _parent field's type option can't be changed: [" + parentType + "]->["
+                + fieldMergeWith.parentType + "]");
         }
         // If fieldMergeWith is not active it means the user provided a mapping
         // update that does not explicitly configure the _parent field, so we
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
index 2e2898d781fb5..0d281d0f0a5fe 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -55,7 +55,8 @@ public static class Defaults {
     public static class Builder extends ObjectMapper.Builder<Builder, RootObjectMapper> {
 
         protected Explicit<DynamicTemplate[]> dynamicTemplates = new Explicit<>(new DynamicTemplate[0], false);
-        protected Explicit<FormatDateTimeFormatter[]> dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
+        protected Explicit<FormatDateTimeFormatter[]> dynamicDateTimeFormatters =
+            new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
         protected Explicit<Boolean> dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
         protected Explicit<Boolean> numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
index 52f55699eb317..b23d1daeabdab 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
@@ -81,7 +81,8 @@ public RoutingFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                      ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index cf45d37392744..f66dde9156d94 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -108,7 +108,8 @@ public SourceFieldMapper build(BuilderContext context) {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                      ParserContext parserContext) throws MapperParsingException {
             Builder builder = new Builder();
 
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index 5bd831d1a8b44..75be214dc5c4d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -81,7 +81,8 @@ public static class Defaults {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                      ParserContext parserContext) throws MapperParsingException {
             throw new MapperParsingException(NAME + " is not configurable");
         }
 
@@ -187,7 +188,8 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
             } else {
                 // this means the index has a single type and the type field is implicit
                 DEPRECATION_LOGGER.deprecatedAndMaybeLog("range_single_type",
-                        "Running [range] query on [_type] field for an index with a single type. As types are deprecated, this functionality will be removed in future releases.");
+                        "Running [range] query on [_type] field for an index with a single type. As types are deprecated,"
+                            + " this functionality will be removed in future releases.");
                 Collection<String> types = context.getMapperService().types();
                 String type = types.iterator().hasNext() ? types.iterator().next() : null;
                 if (type != null) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
index 43fee345ffa33..cb6ec550cff89 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
@@ -80,7 +80,8 @@ public static class Defaults {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node,
+                                                       ParserContext parserContext) throws MapperParsingException {
             throw new MapperParsingException(NAME + " is not configurable");
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
index 9b33c5c84f714..a98224c76e06b 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
@@ -58,7 +58,8 @@ public static class Defaults {
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node,
+                                                       ParserContext parserContext) throws MapperParsingException {
             throw new MapperParsingException(NAME + " is not configurable");
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index b14fab84130b1..218b20ecad39c 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -309,18 +309,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             query = new MatchAllDocsQuery();
         }
 
+        CombineFunction boostMode = this.boostMode == null ? DEFAULT_BOOST_MODE : this.boostMode;
         // handle cases where only one score function and no filter was provided. In this case we create a FunctionScoreQuery.
         if (filterFunctions.length == 0) {
             return new FunctionScoreQuery(query, minScore, maxBoost);
         } else if (filterFunctions.length == 1 && filterFunctions[0] instanceof FunctionScoreQuery.FilterScoreFunction == false) {
-            CombineFunction combineFunction = this.boostMode;
-            if (combineFunction == null) {
-                combineFunction = filterFunctions[0].getDefaultScoreCombiner();
-            }
-            return new FunctionScoreQuery(query, filterFunctions[0], combineFunction, minScore, maxBoost);
+            return new FunctionScoreQuery(query, filterFunctions[0], boostMode, minScore, maxBoost);
         }
         // in all other cases we create a FunctionScoreQuery with filters
-        CombineFunction boostMode = this.boostMode == null ? DEFAULT_BOOST_MODE : this.boostMode;
         return new FunctionScoreQuery(query, scoreMode, filterFunctions, boostMode, minScore, maxBoost);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java b/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
index c711fb429366e..a56f8670c23b1 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class SeqNoStats implements ToXContentFragment, Writeable {
 
@@ -83,6 +84,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final SeqNoStats that = (SeqNoStats) o;
+        return maxSeqNo == that.maxSeqNo &&
+            localCheckpoint == that.localCheckpoint &&
+            globalCheckpoint == that.globalCheckpoint;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxSeqNo, localCheckpoint, globalCheckpoint);
+    }
+
     @Override
     public String toString() {
         return "SeqNoStats{" +
diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
index 3530fe5ae5db8..683a5a79c36af 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
+
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -62,13 +62,12 @@ public class PrimaryReplicaSyncer extends AbstractComponent {
     private volatile ByteSizeValue chunkSize = DEFAULT_CHUNK_SIZE;
 
     @Inject
-    public PrimaryReplicaSyncer(Settings settings, TransportService transportService, TransportResyncReplicationAction syncAction) {
-        this(settings, transportService.getTaskManager(), syncAction);
+    public PrimaryReplicaSyncer(TransportService transportService, TransportResyncReplicationAction syncAction) {
+        this(transportService.getTaskManager(), syncAction);
     }
 
     // for tests
-    public PrimaryReplicaSyncer(Settings settings, TaskManager taskManager, SyncAction syncAction) {
-        super(settings);
+    public PrimaryReplicaSyncer(TaskManager taskManager, SyncAction syncAction) {
         this.taskManager = taskManager;
         this.syncAction = syncAction;
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index ac5a5047464bf..bf3046c954829 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -94,7 +94,6 @@ public class IndexingMemoryController extends AbstractComponent implements Index
     private final ShardsIndicesStatusChecker statusChecker;
 
     IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard> indexServices) {
-        super(settings);
         this.indexShards = indexServices;
 
         ByteSizeValue indexingBuffer = INDEX_BUFFER_SIZE_SETTING.get(settings);
@@ -103,8 +102,8 @@ public class IndexingMemoryController extends AbstractComponent implements Index
         // null means we used the default (10%)
         if (indexingBufferSetting == null || indexingBufferSetting.endsWith("%")) {
             // We only apply the min/max when % value was used for the index buffer:
-            ByteSizeValue minIndexingBuffer = MIN_INDEX_BUFFER_SIZE_SETTING.get(this.settings);
-            ByteSizeValue maxIndexingBuffer = MAX_INDEX_BUFFER_SIZE_SETTING.get(this.settings);
+            ByteSizeValue minIndexingBuffer = MIN_INDEX_BUFFER_SIZE_SETTING.get(settings);
+            ByteSizeValue maxIndexingBuffer = MAX_INDEX_BUFFER_SIZE_SETTING.get(settings);
             if (indexingBuffer.getBytes() < minIndexingBuffer.getBytes()) {
                 indexingBuffer = minIndexingBuffer;
             }
@@ -114,9 +113,9 @@ public class IndexingMemoryController extends AbstractComponent implements Index
         }
         this.indexingBuffer = indexingBuffer;
 
-        this.inactiveTime = SHARD_INACTIVE_TIME_SETTING.get(this.settings);
+        this.inactiveTime = SHARD_INACTIVE_TIME_SETTING.get(settings);
         // we need to have this relatively small to free up heap quickly enough
-        this.interval = SHARD_MEMORY_INTERVAL_TIME_SETTING.get(this.settings);
+        this.interval = SHARD_MEMORY_INTERVAL_TIME_SETTING.get(settings);
 
         this.statusChecker = new ShardsIndicesStatusChecker();
 
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
index 2695c1728491b..129b839bac75a 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
@@ -71,7 +71,6 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache,
     private final Map<Object, StatsAndCount> stats2 = new IdentityHashMap<>();
 
     public IndicesQueryCache(Settings settings) {
-        super(settings);
         final ByteSizeValue size = INDICES_CACHE_QUERY_SIZE_SETTING.get(settings);
         final int count = INDICES_CACHE_QUERY_COUNT_SETTING.get(settings);
         logger.debug("using [node] query cache with size [{}] max filter count [{}]",
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 4f2ac5e5b21cd..be7601e645861 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -83,7 +83,6 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
     private final Cache<Key, BytesReference> cache;
 
     IndicesRequestCache(Settings settings) {
-        super(settings);
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
         long sizeInBytes = size.getBytes();
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index d69cd6f167f63..dad034ae51c14 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -173,6 +173,10 @@ public class IndicesService extends AbstractLifecycleComponent
         }
     }
 
+    /**
+     * The node's settings.
+     */
+    private final Settings settings;
     private final PluginsService pluginsService;
     private final NodeEnvironment nodeEnv;
     private final NamedXContentRegistry xContentRegistry;
@@ -215,6 +219,7 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi
                           Collection<Function<IndexSettings, Optional<EngineFactory>>> engineFactoryProviders,
                           Map<String, Function<IndexSettings, IndexStore>> indexStoreFactories) {
         super(settings);
+        this.settings = settings;
         this.threadPool = threadPool;
         this.pluginsService = pluginsService;
         this.nodeEnv = nodeEnv;
@@ -483,7 +488,7 @@ private synchronized IndexService createIndexService(final String reason,
                                                          IndicesFieldDataCache indicesFieldDataCache,
                                                          List<IndexEventListener> builtInListeners,
                                                          IndexingOperationListener... indexingOperationListeners) throws IOException {
-        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopedSettings);
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, settings, indexScopedSettings);
         // we ignore private settings since they are not registered settings
         indexScopedSettings.validate(indexMetaData.getSettings(), true, true, true);
         logger.debug("creating Index [{}], shards [{}]/[{}] - reason [{}]",
diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index 4a55b86291e63..8c2eb9b67b8df 100644
--- a/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/server/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -90,7 +90,6 @@ public class HunspellService extends AbstractComponent {
 
     public HunspellService(final Settings settings, final Environment env, final Map<String, Dictionary> knownDictionaries)
             throws IOException {
-        super(settings);
         this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries);
         this.hunspellDir = resolveHunspellDirectory(env);
         this.defaultIgnoreCase = HUNSPELL_IGNORE_CASE.get(settings);
diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 561a8e8c74290..cd32c415ea54b 100644
--- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -107,6 +107,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
     private static final ShardStateAction.Listener SHARD_STATE_ACTION_LISTENER = new ShardStateAction.Listener() {
     };
 
+    private final Settings settings;
     // a list of shards that failed during recovery
     // we keep track of these shards in order to prevent repeated recovery of these shards on each cluster state update
     final ConcurrentMap<ShardId, ShardRouting> failedShardsCache = ConcurrentCollections.newConcurrentMap();
@@ -156,6 +157,7 @@ public IndicesClusterStateService(Settings settings,
                                PrimaryReplicaSyncer primaryReplicaSyncer,
                                Consumer<ShardId> globalCheckpointSyncer) {
         super(settings);
+        this.settings = settings;
         this.buildInIndexListener =
                 Arrays.asList(
                         peerRecoverySourceService,
@@ -172,7 +174,7 @@ public IndicesClusterStateService(Settings settings,
         this.repositoriesService = repositoriesService;
         this.primaryReplicaSyncer = primaryReplicaSyncer;
         this.globalCheckpointSyncer = globalCheckpointSyncer;
-        this.sendRefreshMapping = this.settings.getAsBoolean("indices.cluster.send_refresh_mapping", true);
+        this.sendRefreshMapping = settings.getAsBoolean("indices.cluster.send_refresh_mapping", true);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
index 4a784af6bb3f3..a5945187de154 100644
--- a/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/server/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -58,7 +58,6 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
     private final Cache<Key, Accountable> cache;
 
     public IndicesFieldDataCache(Settings settings, IndexFieldDataCache.Listener indicesFieldDataCacheListener) {
-        super(settings);
         this.indicesFieldDataCacheListener = indicesFieldDataCacheListener;
         final long sizeInBytes = INDICES_FIELDDATA_CACHE_SIZE_KEY.get(settings).getBytes();
         CacheBuilder<Key, Accountable> cacheBuilder = CacheBuilder.<Key, Accountable>builder()
diff --git a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index f40a4ef69f22c..45634d7168f5f 100644
--- a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.Index;
@@ -83,8 +82,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
-    public SyncedFlushService(Settings settings, IndicesService indicesService, ClusterService clusterService, TransportService transportService, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings);
+    public SyncedFlushService(IndicesService indicesService, ClusterService clusterService, TransportService transportService, IndexNameExpressionResolver indexNameExpressionResolver) {
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.transportService = transportService;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
index ca9f4cc22f3c6..2cf514b114bb9 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
@@ -60,9 +60,8 @@ public static class Actions {
     final OngoingRecoveries ongoingRecoveries = new OngoingRecoveries();
 
     @Inject
-    public PeerRecoverySourceService(Settings settings, TransportService transportService, IndicesService indicesService,
+    public PeerRecoverySourceService(TransportService transportService, IndicesService indicesService,
                                      RecoverySettings recoverySettings) {
-        super(settings);
         this.transportService = transportService;
         this.indicesService = indicesService;
         this.recoverySettings = recoverySettings;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index cefc196b73a73..60e327df03164 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -100,9 +100,8 @@ public static class Actions {
 
     private final RecoveriesCollection onGoingRecoveries;
 
-    public PeerRecoveryTargetService(Settings settings, ThreadPool threadPool, TransportService transportService, RecoverySettings
-            recoverySettings, ClusterService clusterService) {
-        super(settings);
+    public PeerRecoveryTargetService(ThreadPool threadPool, TransportService transportService,
+            RecoverySettings recoverySettings, ClusterService clusterService) {
         this.threadPool = threadPool;
         this.transportService = transportService;
         this.recoverySettings = recoverySettings;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
index e238277b698d0..b90bed90d052b 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
@@ -85,8 +85,6 @@ public class RecoverySettings extends AbstractComponent {
     private volatile ByteSizeValue chunkSize = DEFAULT_CHUNK_SIZE;
 
     public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
-        super(settings);
-
         this.retryDelayStateSync = INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.get(settings);
         // doesn't have to be fast as nodes are reconnected every 10s by default (see InternalClusterService.ReconnectToNodes)
         // and we want to give the master time to remove a faulty node
diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 29f6e7aeeecc1..567785fd23209 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -77,6 +77,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
             Property.NodeScope);
     public static final String ACTION_SHARD_EXISTS = "internal:index/shard/exists";
     private static final EnumSet<IndexShardState> ACTIVE_STATES = EnumSet.of(IndexShardState.STARTED);
+    private final Settings settings;
     private final IndicesService indicesService;
     private final ClusterService clusterService;
     private final TransportService transportService;
@@ -90,7 +91,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
     @Inject
     public IndicesStore(Settings settings, IndicesService indicesService,
                         ClusterService clusterService, TransportService transportService, ThreadPool threadPool) {
-        super(settings);
+        this.settings = settings;
         this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.transportService = transportService;
diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index ac6f7aff8e3d3..241066e840731 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -67,6 +67,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
     public static final String ACTION_NAME = "internal:cluster/nodes/indices/shard/store";
 
+    private final Settings settings;
     private final IndicesService indicesService;
     private final NodeEnvironment nodeEnv;
     private final NamedXContentRegistry namedXContentRegistry;
@@ -79,6 +80,7 @@ public TransportNodesListShardStoreMetaData(Settings settings, ThreadPool thread
                                                 NamedXContentRegistry namedXContentRegistry) {
         super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
             Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
+        this.settings = settings;
         this.indicesService = indicesService;
         this.nodeEnv = nodeEnv;
         this.namedXContentRegistry = namedXContentRegistry;
diff --git a/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
index cacba54d80ad4..16ab96be06786 100644
--- a/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
+++ b/server/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeEnvironment.NodePath;
 
@@ -45,8 +44,7 @@ public class FsProbe extends AbstractComponent {
 
     private final NodeEnvironment nodeEnv;
 
-    public FsProbe(Settings settings, NodeEnvironment nodeEnv) {
-        super(settings);
+    public FsProbe(NodeEnvironment nodeEnv) {
         this.nodeEnv = nodeEnv;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/server/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index 66534514b361e..66058c9f79c30 100644
--- a/server/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/server/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -48,8 +48,7 @@ public class FsService extends AbstractComponent {
             Property.NodeScope);
 
     public FsService(final Settings settings, final NodeEnvironment nodeEnvironment, ClusterInfoService clusterInfoService) {
-        super(settings);
-        this.probe = new FsProbe(settings, nodeEnvironment);
+        this.probe = new FsProbe(nodeEnvironment);
         this.clusterInfoService = clusterInfoService;
         refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         logger.debug("using refresh_interval [{}]", refreshInterval);
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
index f9d9fd80be094..eb6bd6f2f56b4 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
@@ -38,7 +38,6 @@ public class JvmService extends AbstractComponent {
             Property.NodeScope);
 
     public JvmService(Settings settings) {
-        super(settings);
         this.jvmInfo = JvmInfo.jvmInfo();
         this.jvmStats = JvmStats.jvmStats();
 
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsService.java b/server/src/main/java/org/elasticsearch/monitor/os/OsService.java
index ef6270850a747..8812c1f25b766 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsService.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsService.java
@@ -38,7 +38,6 @@ public class OsService extends AbstractComponent {
                 Property.NodeScope);
 
     public OsService(Settings settings) {
-        super(settings);
         this.probe = OsProbe.getInstance();
         TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         this.info = probe.osInfo(refreshInterval.millis(), EsExecutors.numberOfProcessors(settings));
diff --git a/server/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/server/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
index 7bebeb975863f..aba7993850af5 100644
--- a/server/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
+++ b/server/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
@@ -37,7 +37,6 @@ public final class ProcessService extends AbstractComponent {
             Property.NodeScope);
 
     public ProcessService(Settings settings) {
-        super(settings);
         this.probe = ProcessProbe.getInstance();
         final TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         processStatsCache = new ProcessStatsCache(refreshInterval, probe.processStats());
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 5ae085f5fbff8..a8825e2221e35 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -388,7 +388,7 @@ protected Node(
                 clusterService.getClusterSettings(), client);
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client,
                 listener::onNewInfo);
-            final UsageService usageService = new UsageService(settings);
+            final UsageService usageService = new UsageService();
 
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
@@ -430,7 +430,7 @@ protected Node(
                 ClusterModule.getNamedXWriteables().stream())
                 .flatMap(Function.identity()).collect(toList()));
             modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class), xContentRegistry));
-            final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment, xContentRegistry);
+            final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry);
 
             // collect engine factory providers from server and from plugins
             final Collection<EnginePlugin> enginePlugins = pluginsService.filterPlugins(EnginePlugin.class);
@@ -454,7 +454,7 @@ protected Node(
                             threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays,
                             scriptModule.getScriptService(), client, metaStateService, engineFactoryProviders, indexStoreFactories);
 
-            final AliasValidator aliasValidator = new AliasValidator(settings);
+            final AliasValidator aliasValidator = new AliasValidator();
 
             final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService(
                     settings,
@@ -498,7 +498,7 @@ protected Node(
                 indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders);
             final GatewayMetaState gatewayMetaState = new GatewayMetaState(settings, nodeEnvironment, metaStateService,
                 metaDataIndexUpgradeService, metaDataUpgrader);
-            new TemplateUpgradeService(settings, client, clusterService, threadPool, indexTemplateMetaDataUpgraders);
+            new TemplateUpgradeService(client, clusterService, threadPool, indexTemplateMetaDataUpgraders);
             final Transport transport = networkModule.getTransportSupplier().get();
             Set<String> taskHeaders = Stream.concat(
                 pluginsService.filterPlugins(ActionPlugin.class).stream().flatMap(p -> p.getTaskHeaders().stream()),
@@ -506,8 +506,8 @@ protected Node(
             ).collect(Collectors.toSet());
             final TransportService transportService = newTransportService(settings, transport, threadPool,
                 networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings(), taskHeaders);
-            final ResponseCollectorService responseCollectorService = new ResponseCollectorService(this.settings, clusterService);
-            final SearchTransportService searchTransportService =  new SearchTransportService(settings, transportService,
+            final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService);
+            final SearchTransportService searchTransportService =  new SearchTransportService(transportService,
                 SearchExecutionStatsCollector.makeWrapper(responseCollectorService));
             final Consumer<Binder> httpBind;
             final HttpServerTransport httpServerTransport;
@@ -542,10 +542,10 @@ protected Node(
                 .flatMap(List::stream)
                 .collect(toList());
 
-            final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(settings, tasksExecutors);
+            final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(tasksExecutors);
             final PersistentTasksClusterService persistentTasksClusterService =
                 new PersistentTasksClusterService(settings, registry, clusterService);
-            final PersistentTasksService persistentTasksService = new PersistentTasksService(settings, clusterService, threadPool, client);
+            final PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client);
 
             modules.add(b -> {
                     b.bind(Node.class).toInstance(this);
@@ -572,12 +572,11 @@ protected Node(
                     b.bind(MetaDataCreateIndexService.class).toInstance(metaDataCreateIndexService);
                     b.bind(SearchService.class).toInstance(searchService);
                     b.bind(SearchTransportService.class).toInstance(searchTransportService);
-                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings,
-                        searchService::createReduceContext));
+                    b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(searchService::createReduceContext));
                     b.bind(Transport.class).toInstance(transport);
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
-                    b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
+                    b.bind(UpdateHelper.class).toInstance(new UpdateHelper(scriptModule.getScriptService()));
                     b.bind(MetaDataIndexUpgradeService.class).toInstance(metaDataIndexUpgradeService);
                     b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
                     b.bind(GatewayMetaState.class).toInstance(gatewayMetaState);
@@ -585,9 +584,9 @@ protected Node(
                     {
                         RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings());
                         processRecoverySettings(settingsModule.getClusterSettings(), recoverySettings);
-                        b.bind(PeerRecoverySourceService.class).toInstance(new PeerRecoverySourceService(settings, transportService,
+                        b.bind(PeerRecoverySourceService.class).toInstance(new PeerRecoverySourceService(transportService,
                                 indicesService, recoverySettings));
-                        b.bind(PeerRecoveryTargetService.class).toInstance(new PeerRecoveryTargetService(settings, threadPool,
+                        b.bind(PeerRecoveryTargetService.class).toInstance(new PeerRecoveryTargetService(threadPool,
                                 transportService, recoverySettings, clusterService));
                     }
                     httpBind.accept(b);
diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java
index 207886c5cf263..fe9c3d59d6646 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeService.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeService.java
@@ -46,7 +46,7 @@
 import java.io.IOException;
 
 public class NodeService extends AbstractComponent implements Closeable {
-
+    private final Settings settings;
     private final ThreadPool threadPool;
     private final MonitorService monitorService;
     private final TransportService transportService;
@@ -68,7 +68,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService,
                 SettingsFilter settingsFilter, ResponseCollectorService responseCollectorService,
                 SearchTransportService searchTransportService) {
-        super(settings);
+        this.settings = settings;
         this.threadPool = threadPool;
         this.monitorService = monitorService;
         this.transportService = transportService;
diff --git a/server/src/main/java/org/elasticsearch/node/ResponseCollectorService.java b/server/src/main/java/org/elasticsearch/node/ResponseCollectorService.java
index 4ab1bc4cee80b..8885728927b34 100644
--- a/server/src/main/java/org/elasticsearch/node/ResponseCollectorService.java
+++ b/server/src/main/java/org/elasticsearch/node/ResponseCollectorService.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 
 import java.io.IOException;
@@ -49,8 +48,7 @@ public final class ResponseCollectorService extends AbstractComponent implements
 
     private final ConcurrentMap<String, NodeStatistics> nodeIdToStats = ConcurrentCollections.newConcurrentMap();
 
-    public ResponseCollectorService(Settings settings, ClusterService clusterService) {
-        super(settings);
+    public ResponseCollectorService(ClusterService clusterService) {
         clusterService.addListener(this);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
index 9ed0af010b530..dd4bd30b19f4a 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
@@ -48,7 +48,6 @@ public class PersistentTasksClusterService extends AbstractComponent implements
     private final EnableAssignmentDecider decider;
 
     public PersistentTasksClusterService(Settings settings, PersistentTasksExecutorRegistry registry, ClusterService clusterService) {
-        super(settings);
         this.clusterService = clusterService;
         clusterService.addListener(this);
         this.registry = registry;
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java
index 758ffbe69a04d..a02efb6805729 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
 import org.elasticsearch.tasks.TaskId;
@@ -40,8 +39,7 @@ public abstract class PersistentTasksExecutor<Params extends PersistentTaskParam
     private final String executor;
     private final String taskName;
 
-    protected PersistentTasksExecutor(Settings settings, String taskName, String executor) {
-        super(settings);
+    protected PersistentTasksExecutor(String taskName, String executor) {
         this.taskName = taskName;
         this.executor = executor;
     }
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
index 2ac57e074b7bf..0a5f7d8bfbb10 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
@@ -18,9 +18,6 @@
  */
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
-
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -29,13 +26,12 @@
 /**
  * Components that registers all persistent task executors
  */
-public class PersistentTasksExecutorRegistry extends AbstractComponent {
+public class PersistentTasksExecutorRegistry {
 
     private final Map<String, PersistentTasksExecutor<?>> taskExecutors;
 
     @SuppressWarnings("unchecked")
-    public PersistentTasksExecutorRegistry(Settings settings, Collection<PersistentTasksExecutor<?>> taskExecutors) {
-        super(settings);
+    public PersistentTasksExecutorRegistry(Collection<PersistentTasksExecutor<?>> taskExecutors) {
         Map<String, PersistentTasksExecutor<?>> map = new HashMap<>();
         for (PersistentTasksExecutor<?> executor : taskExecutors) {
             map.put(executor.getTaskName(), executor);
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
index 91cdb400aa0d4..a90415b530b43 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.tasks.Task;
@@ -57,11 +56,9 @@ public class PersistentTasksNodeService extends AbstractComponent implements Clu
     private final TaskManager taskManager;
     private final NodePersistentTasksExecutor nodePersistentTasksExecutor;
 
-    public PersistentTasksNodeService(Settings settings,
-                                      PersistentTasksService persistentTasksService,
+    public PersistentTasksNodeService(PersistentTasksService persistentTasksService,
                                       PersistentTasksExecutorRegistry persistentTasksExecutorRegistry,
                                       TaskManager taskManager, NodePersistentTasksExecutor nodePersistentTasksExecutor) {
-        super(settings);
         this.persistentTasksService = persistentTasksService;
         this.persistentTasksExecutorRegistry = persistentTasksExecutorRegistry;
         this.taskManager = taskManager;
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
index c820b244af2db..ac80d1888bcc7 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.node.NodeClosedException;
@@ -56,8 +55,7 @@ public class PersistentTasksService extends AbstractComponent {
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
 
-    public PersistentTasksService(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) {
-        super(settings);
+    public PersistentTasksService(ClusterService clusterService, ThreadPool threadPool, Client client) {
         this.client = client;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
index 341fdc5062200..c59e896fb5141 100644
--- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
@@ -210,7 +210,7 @@ public TransportAction(Settings settings, TransportService transportService, Clu
                     indexNameExpressionResolver, Request::new);
             this.persistentTasksClusterService = persistentTasksClusterService;
             NodePersistentTasksExecutor executor = new NodePersistentTasksExecutor(threadPool);
-            clusterService.addListener(new PersistentTasksNodeService(settings, persistentTasksService, persistentTasksExecutorRegistry,
+            clusterService.addListener(new PersistentTasksNodeService(persistentTasksService, persistentTasksExecutorRegistry,
                     transportService.getTaskManager(), executor));
         }
 
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 4ce489268714f..5dfdeb09095e3 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -72,6 +72,7 @@
 
 public class PluginsService extends AbstractComponent {
 
+    private final Settings settings;
     private final Path configPath;
 
     /**
@@ -79,6 +80,7 @@ public class PluginsService extends AbstractComponent {
      */
     private final List<Tuple<PluginInfo, Plugin>> plugins;
     private final PluginsAndModules info;
+
     public static final Setting<List<String>> MANDATORY_SETTING =
         Setting.listSetting("plugin.mandatory", Collections.emptyList(), Function.identity(), Property.NodeScope);
 
@@ -98,8 +100,7 @@ public List<String> getPluginSettingsFilter() {
      * @param classpathPlugins Plugins that exist in the classpath which should be loaded
      */
     public PluginsService(Settings settings, Path configPath, Path modulesDirectory, Path pluginsDirectory, Collection<Class<? extends Plugin>> classpathPlugins) {
-        super(settings);
-
+        this.settings = settings;
         this.configPath = configPath;
 
         List<Tuple<PluginInfo, Plugin>> pluginsLoaded = new ArrayList<>();
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index aef4381cd8b2d..e97f7acf168f7 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -69,7 +69,6 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
     public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService,
                                Map<String, Repository.Factory> typesRegistry,
                                ThreadPool threadPool) {
-        super(settings);
         this.typesRegistry = typesRegistry;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -78,7 +77,7 @@ public RepositoriesService(Settings settings, ClusterService clusterService, Tra
         if (DiscoveryNode.isDataNode(settings) || DiscoveryNode.isMasterNode(settings)) {
             clusterService.addStateApplier(this);
         }
-        this.verifyAction = new VerifyNodeRepositoryAction(settings, transportService, clusterService, this);
+        this.verifyAction = new VerifyNodeRepositoryAction(transportService, clusterService, this);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 94a289fb10207..5c14c5a5781be 100644
--- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoriesService.VerifyResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
@@ -55,8 +54,7 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
 
     private final RepositoriesService repositoriesService;
 
-    public VerifyNodeRepositoryAction(Settings settings, TransportService transportService, ClusterService clusterService, RepositoriesService repositoriesService) {
-        super(settings);
+    public VerifyNodeRepositoryAction(TransportService transportService, ClusterService clusterService, RepositoriesService repositoriesService) {
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.repositoriesService = repositoriesService;
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 983a9e25592c1..3e43868072cd7 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -205,6 +205,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String DATA_BLOB_PREFIX = "__";
 
+    private final Settings settings;
+
     private final RateLimiter snapshotRateLimiter;
 
     private final RateLimiter restoreRateLimiter;
@@ -235,10 +237,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * Constructs new BlobStoreRepository
      *
      * @param metadata       The metadata for this repository including name and settings
-     * @param globalSettings Settings for the node this repository object is created on
+     * @param settings Settings for the node this repository object is created on
      */
-    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings, NamedXContentRegistry namedXContentRegistry) {
-        super(globalSettings);
+    protected BlobStoreRepository(RepositoryMetaData metadata, Settings settings, NamedXContentRegistry namedXContentRegistry) {
+        super(settings);
+        this.settings = settings;
         this.metadata = metadata;
         this.namedXContentRegistry = namedXContentRegistry;
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 643ff2bc93d3c..7abddafac4ed7 100644
--- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -99,10 +99,10 @@ public FsRepository(RepositoryMetaData metadata, Environment environment,
         if (CHUNK_SIZE_SETTING.exists(metadata.settings())) {
             this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings());
         } else {
-            this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(settings);
+            this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(environment.settings());
         }
         this.compress = COMPRESS_SETTING.exists(metadata.settings())
-            ? COMPRESS_SETTING.get(metadata.settings()) : REPOSITORIES_COMPRESS_SETTING.get(settings);
+            ? COMPRESS_SETTING.get(metadata.settings()) : REPOSITORIES_COMPRESS_SETTING.get(environment.settings());
         this.basePath = BlobPath.cleanPath();
     }
 
@@ -110,7 +110,7 @@ public FsRepository(RepositoryMetaData metadata, Environment environment,
     protected BlobStore createBlobStore() throws Exception {
         final String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         final Path locationFile = environment.resolveRepoFile(location);
-        return new FsBlobStore(settings, locationFile);
+        return new FsBlobStore(environment.settings(), locationFile);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index 585713b641f5e..4a982ac80a741 100644
--- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -59,7 +59,7 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH
     private final LongAdder usageCount = new LongAdder();
 
     protected BaseRestHandler(Settings settings) {
-        super(settings);
+        // TODO drop settings from ctor
     }
 
     public final long getUsageCount() {
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index aae63f041fad5..98dc631324c8f 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -75,9 +74,8 @@ public class RestController extends AbstractComponent implements HttpServerTrans
     private final Set<String> headersToCopy;
     private UsageService usageService;
 
-    public RestController(Settings settings, Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
+    public RestController(Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
             NodeClient client, CircuitBreakerService circuitBreakerService, UsageService usageService) {
-        super(settings);
         this.headersToCopy = headersToCopy;
         this.usageService = usageService;
         if (handlerWrapper == null) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
index 746bb643bf62d..e1e4d92116349 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
@@ -43,12 +43,14 @@
 
 public class RestClusterGetSettingsAction extends BaseRestHandler {
 
+    private final Settings settings;
     private final ClusterSettings clusterSettings;
     private final SettingsFilter settingsFilter;
 
     public RestClusterGetSettingsAction(Settings settings, RestController controller, ClusterSettings clusterSettings,
             SettingsFilter settingsFilter) {
         super(settings);
+        this.settings = settings;
         this.clusterSettings = clusterSettings;
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/settings", this);
         this.settingsFilter = settingsFilter;
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
index dc2474c653315..d9f746a029ca3 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
@@ -50,7 +50,7 @@ public String getName() {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         String scrollIds = request.param("scroll_id");
         ClearScrollRequest clearRequest = new ClearScrollRequest();
-        clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
+        clearRequest.setScrollIds(Arrays.asList(Strings.splitStringByCommaToArray(scrollIds)));
         request.withContentOrSourceParamParserOrNull((xContentParser -> {
             if (xContentParser != null) {
                 // NOTE: if rest request with xcontent body has request parameters, values parsed from request body have the precedence
@@ -65,10 +65,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel));
     }
 
-    private static String[] splitScrollIds(String scrollIds) {
-        if (scrollIds == null) {
-            return Strings.EMPTY_ARRAY;
-        }
-        return Strings.splitStringByCommaToArray(scrollIds);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java
index 8927e13b16962..eeedd6f1d5b0f 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptMetaData.java
@@ -68,7 +68,7 @@ public static final class Builder {
          *                 is no existing {@link ScriptMetaData}.
          */
         public Builder(ScriptMetaData previous) {
-            this.scripts = previous == null ? new HashMap<>() :new HashMap<>(previous.scripts);
+            this.scripts = previous == null ? new HashMap<>() : new HashMap<>(previous.scripts);
         }
 
         /**
@@ -393,6 +393,13 @@ public EnumSet<MetaData.XContentContext> context() {
         return MetaData.ALL_CONTEXTS;
     }
 
+    /**
+     * Returns the map of stored scripts.
+     */
+    Map<String, StoredScriptSource> getStoredScripts() {
+        return scripts;
+    }
+
     /**
      * Retrieves a stored script based on a user-specified id.
      */
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java
index 30c34beecaf19..6838231146cb1 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -48,6 +48,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -97,8 +98,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
     public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
         Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES =
-        Setting.intSetting("script.max_size_in_bytes", 65535, Property.NodeScope);
-    // public Setting(String key, Function<Settings, String> defaultValue, Function<String, T> parser, Property... properties) {
+        Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope);
     public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE =
             new Setting<>("script.max_compilations_rate", "75/5m", MAX_COMPILATION_RATE_FUNCTION, Property.Dynamic, Property.NodeScope);
 
@@ -109,6 +109,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
     public static final Setting<List<String>> CONTEXTS_ALLOWED_SETTING =
         Setting.listSetting("script.allowed_contexts", Collections.emptyList(), Function.identity(), Setting.Property.NodeScope);
 
+    private final Settings settings;
     private final Set<String> typesAllowed;
     private final Set<String> contextsAllowed;
 
@@ -121,15 +122,15 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
 
     private ClusterState clusterState;
 
+    private int maxSizeInBytes;
+
     private Tuple<Integer, TimeValue> rate;
     private long lastInlineCompileTime;
     private double scriptsPerTimeWindow;
     private double compilesAllowedPerNano;
 
     public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) {
-        super(settings);
-
-        Objects.requireNonNull(settings);
+        this.settings = Objects.requireNonNull(settings);
         this.engines = Objects.requireNonNull(engines);
         this.contexts = Objects.requireNonNull(contexts);
 
@@ -221,10 +222,12 @@ public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<S
         this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
 
         this.lastInlineCompileTime = System.nanoTime();
+        this.setMaxSizeInBytes(SCRIPT_MAX_SIZE_IN_BYTES.get(settings));
         this.setMaxCompilationRate(SCRIPT_MAX_COMPILATIONS_RATE.get(settings));
     }
 
     void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
+        clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_SIZE_IN_BYTES, this::setMaxSizeInBytes);
         clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_COMPILATIONS_RATE, this::setMaxCompilationRate);
     }
 
@@ -241,6 +244,22 @@ private ScriptEngine getEngine(String lang) {
         return scriptEngine;
     }
 
+    /**
+     * Changes the maximum number of bytes a script's source is allowed to have.
+     * @param newMaxSizeInBytes The new maximum number of bytes.
+     */
+    void setMaxSizeInBytes(int newMaxSizeInBytes) {
+        for (Map.Entry<String, StoredScriptSource> source : getScriptsFromClusterState().entrySet()) {
+            if (source.getValue().getSource().getBytes(StandardCharsets.UTF_8).length > newMaxSizeInBytes) {
+                throw new IllegalArgumentException("script.max_size_in_bytes cannot be set to [" + newMaxSizeInBytes + "], " +
+                        "stored script [" + source.getKey() + "] exceeds the new value with a size of " +
+                        "[" + source.getValue().getSource().getBytes(StandardCharsets.UTF_8).length + "]");
+            }
+        }
+
+        maxSizeInBytes = newMaxSizeInBytes;
+    }
+
     /**
      * This configures the maximum script compilations per five minute window.
      *
@@ -304,6 +323,13 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
             throw new IllegalArgumentException("cannot execute scripts using [" + context.name + "] context");
         }
 
+        if (type == ScriptType.INLINE) {
+            if (idOrCode.getBytes(StandardCharsets.UTF_8).length > maxSizeInBytes) {
+                throw new IllegalArgumentException("exceeded max allowed inline script size in bytes [" + maxSizeInBytes + "] " +
+                        "with size [" + idOrCode.getBytes(StandardCharsets.UTF_8).length + "] for script [" + idOrCode + "]");
+            }
+        }
+
         if (logger.isTraceEnabled()) {
             logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode);
         }
@@ -398,6 +424,20 @@ public boolean isAnyContextEnabled() {
         return contextsAllowed == null || contextsAllowed.isEmpty() == false;
     }
 
+    Map<String, StoredScriptSource> getScriptsFromClusterState() {
+        if (clusterState == null) {
+            return Collections.emptyMap();
+        }
+
+        ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE);
+
+        if (scriptMetadata == null) {
+            return Collections.emptyMap();
+        }
+
+        return scriptMetadata.getStoredScripts();
+    }
+
     StoredScriptSource getScriptFromClusterState(String id) {
         ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE);
 
@@ -416,10 +456,8 @@ StoredScriptSource getScriptFromClusterState(String id) {
 
     public void putStoredScript(ClusterService clusterService, PutStoredScriptRequest request,
                                 ActionListener<AcknowledgedResponse> listener) {
-        int max = SCRIPT_MAX_SIZE_IN_BYTES.get(settings);
-
-        if (request.content().length() > max) {
-            throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + max + "] with size [" +
+        if (request.content().length() > maxSizeInBytes) {
+            throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + maxSizeInBytes + "] with size [" +
                 request.content().length() + "] for script [" + request.id() + "]");
         }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
index 0c34c59b7be58..5eb6c62f511bf 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
@@ -22,42 +22,35 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public class ScriptedMetricAggContexts {
-    private static final DeprecationLogger DEPRECATION_LOGGER =
-        new DeprecationLogger(Loggers.getLogger(ScriptedMetricAggContexts.class));
 
-    // Public for access from tests
-    public static final String AGG_PARAM_DEPRECATION_WARNING =
-        "params._agg/_aggs for scripted metric aggregations are deprecated, use state/states (not in params) instead. " +
-        "Use -Des.aggregations.enable_scripted_metric_agg_param=false to disable.";
-
-    public static boolean deprecatedAggParamEnabled() {
-        boolean enabled = Boolean.parseBoolean(
-            System.getProperty("es.aggregations.enable_scripted_metric_agg_param", "true"));
-
-        if (enabled) {
-            DEPRECATION_LOGGER.deprecatedAndMaybeLog("enable_scripted_metric_agg_param", AGG_PARAM_DEPRECATION_WARNING);
+    public abstract static class InitScript {
+        private static final Map<String, String> DEPRECATIONS;
+        static {
+            Map<String, String> deprecations = new HashMap<>();
+            deprecations.put(
+                "_agg",
+                "Accessing variable [_agg] via [params._agg] from within a scripted metric agg init script " +
+                    "is deprecated in favor of using [state]."
+            );
+            DEPRECATIONS = Collections.unmodifiableMap(deprecations);
         }
 
-        return enabled;
-    }
-
-    private abstract static class ParamsAndStateBase {
         private final Map<String, Object> params;
         private final Object state;
 
-        ParamsAndStateBase(Map<String, Object> params, Object state) {
-            this.params = params;
+        public InitScript(Map<String, Object> params, Object state) {
+            this.params = new ParameterMap(params, DEPRECATIONS);
             this.state = state;
         }
 
@@ -68,12 +61,6 @@ public Map<String, Object> getParams() {
         public Object getState() {
             return state;
         }
-    }
-
-    public abstract static class InitScript extends ParamsAndStateBase {
-        public InitScript(Map<String, Object> params, Object state) {
-            super(params, state);
-        }
 
         public abstract void execute();
 
@@ -85,14 +72,51 @@ public interface Factory {
         public static ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_init", Factory.class);
     }
 
-    public abstract static class MapScript extends ParamsAndStateBase {
+    public abstract static class MapScript {
+        private static final Map<String, String> DEPRECATIONS;
+
+        static {
+            Map<String, String> deprecations = new HashMap<>();
+            deprecations.put(
+                "doc",
+                "Accessing variable [doc] via [params.doc] from within a scripted metric agg map script " +
+                    "is deprecated in favor of directly accessing [doc]."
+            );
+            deprecations.put(
+                "_doc",
+                "Accessing variable [doc] via [params._doc] from within a scripted metric agg map script " +
+                    "is deprecated in favor of directly accessing [doc]."
+            );
+            deprecations.put(
+                "_agg",
+                "Accessing variable [_agg] via [params._agg] from within a scripted metric agg map script " +
+                    "is deprecated in favor of using [state]."
+            );
+            DEPRECATIONS = Collections.unmodifiableMap(deprecations);
+        }
+
+        private final Map<String, Object> params;
+        private final Object state;
         private final LeafSearchLookup leafLookup;
         private Scorer scorer;
 
         public MapScript(Map<String, Object> params, Object state, SearchLookup lookup, LeafReaderContext leafContext) {
-            super(params, state);
-
+            this.state = state;
             this.leafLookup = leafContext == null ? null : lookup.getLeafSearchLookup(leafContext);
+            if (leafLookup != null) {
+                params = new HashMap<>(params); // copy params so we aren't modifying input
+                params.putAll(leafLookup.asMap()); // add lookup vars
+                params = new ParameterMap(params, DEPRECATIONS); // wrap with deprecations
+            }
+            this.params = params;
+        }
+
+        public Map<String, Object> getParams() {
+            return params;
+        }
+
+        public Object getState() {
+            return state;
         }
 
         // Return the doc as a map (instead of LeafDocLookup) in order to abide by type whitelisting rules for
@@ -138,9 +162,32 @@ public interface Factory {
         public static ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs_map", Factory.class);
     }
 
-    public abstract static class CombineScript extends ParamsAndStateBase {
+    public abstract static class CombineScript {
+        private static final Map<String, String> DEPRECATIONS;
+        static {
+            Map<String, String> deprecations = new HashMap<>();
+            deprecations.put(
+                "_agg",
+                "Accessing variable [_agg] via [params._agg] from within a scripted metric agg combine script " +
+                    "is deprecated in favor of using [state]."
+            );
+            DEPRECATIONS = Collections.unmodifiableMap(deprecations);
+        }
+
+        private final Map<String, Object> params;
+        private final Object state;
+
         public CombineScript(Map<String, Object> params, Object state) {
-            super(params, state);
+            this.params = new ParameterMap(params, DEPRECATIONS);
+            this.state = state;
+        }
+
+        public Map<String, Object> getParams() {
+            return params;
+        }
+
+        public Object getState() {
+            return state;
         }
 
         public abstract Object execute();
@@ -154,11 +201,22 @@ public interface Factory {
     }
 
     public abstract static class ReduceScript {
+        private static final Map<String, String> DEPRECATIONS;
+        static {
+            Map<String, String> deprecations = new HashMap<>();
+            deprecations.put(
+                "_aggs",
+                "Accessing variable [_aggs] via [params._aggs] from within a scripted metric agg reduce script " +
+                    "is deprecated in favor of using [state]."
+            );
+            DEPRECATIONS = Collections.unmodifiableMap(deprecations);
+        }
+
         private final Map<String, Object> params;
         private final List<Object> states;
 
         public ReduceScript(Map<String, Object> params, List<Object> states) {
-            this.params = params;
+            this.params = new ParameterMap(params, DEPRECATIONS);
             this.states = states;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 4ff6c1ef7bf20..5565a1ec40149 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -183,6 +183,8 @@
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.weighted_avg.InternalWeightedAvg;
 import org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.InternalMedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
@@ -360,6 +362,9 @@ private void registerAggregations(List<SearchPlugin> plugins) {
                 PercentileRanksAggregationBuilder::parse)
                         .addResultReader(InternalTDigestPercentileRanks.NAME, InternalTDigestPercentileRanks::new)
                         .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new));
+        registerAggregation(new AggregationSpec(MedianAbsoluteDeviationAggregationBuilder.NAME,
+                MedianAbsoluteDeviationAggregationBuilder::new, MedianAbsoluteDeviationAggregationBuilder::parse)
+                        .addResultReader(InternalMedianAbsoluteDeviation::new));
         registerAggregation(new AggregationSpec(CardinalityAggregationBuilder.NAME, CardinalityAggregationBuilder::new,
                 CardinalityAggregationBuilder::parse).addResultReader(InternalCardinality::new));
         registerAggregation(new AggregationSpec(GlobalAggregationBuilder.NAME, GlobalAggregationBuilder::new,
@@ -702,7 +707,7 @@ private void registerFetchSubPhases(List<SearchPlugin> plugins) {
         registerFetchSubPhase(new FetchSourceSubPhase());
         registerFetchSubPhase(new VersionFetchSubPhase());
         registerFetchSubPhase(new MatchedQueriesFetchSubPhase());
-        registerFetchSubPhase(new HighlightPhase(settings, highlighters));
+        registerFetchSubPhase(new HighlightPhase(highlighters));
         registerFetchSubPhase(new ParentFieldSubFetchPhase());
 
         FetchPhaseConstructionContext context = new FetchPhaseConstructionContext(highlighters);
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index 978dee84f6f72..bc14331d75c42 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -183,13 +184,14 @@ public SearchService(ClusterService clusterService, IndicesService indicesServic
                          ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, FetchPhase fetchPhase,
                          ResponseCollectorService responseCollectorService) {
         super(clusterService.getSettings());
+        Settings settings = clusterService.getSettings();
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.scriptService = scriptService;
         this.responseCollectorService = responseCollectorService;
         this.bigArrays = bigArrays;
-        this.queryPhase = new QueryPhase(settings);
+        this.queryPhase = new QueryPhase();
         this.fetchPhase = fetchPhase;
         this.multiBucketConsumerService = new MultiBucketConsumerService(clusterService, settings);
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index b4e416f4d7789..62329134d393e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -83,6 +83,8 @@
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.weighted_avg.WeightedAvgAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviation;
 
 import java.util.Map;
 
@@ -316,6 +318,13 @@ public static PercentileRanksAggregationBuilder percentileRanks(String name, dou
         return new PercentileRanksAggregationBuilder(name, values);
     }
 
+    /**
+     * Create a new {@link MedianAbsoluteDeviation} aggregation with the given name
+     */
+    public static MedianAbsoluteDeviationAggregationBuilder medianAbsoluteDeviation(String name) {
+        return new MedianAbsoluteDeviationAggregationBuilder(name);
+    }
+
     /**
      * Create a new {@link Cardinality} aggregation with the given name.
      */
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/InternalMedianAbsoluteDeviation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/InternalMedianAbsoluteDeviation.java
new file mode 100644
index 0000000000000..01bc99777c984
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/InternalMedianAbsoluteDeviation.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class InternalMedianAbsoluteDeviation extends InternalNumericMetricsAggregation.SingleValue implements MedianAbsoluteDeviation {
+
+    static double computeMedianAbsoluteDeviation(TDigestState valuesSketch) {
+
+        if (valuesSketch.size() == 0) {
+            return Double.NaN;
+        } else {
+            final double approximateMedian = valuesSketch.quantile(0.5);
+            final TDigestState approximatedDeviationsSketch = new TDigestState(valuesSketch.compression());
+            valuesSketch.centroids().forEach(centroid -> {
+                final double deviation = Math.abs(approximateMedian - centroid.mean());
+                approximatedDeviationsSketch.add(deviation, centroid.count());
+            });
+
+            return approximatedDeviationsSketch.quantile(0.5);
+        }
+    }
+
+    private final TDigestState valuesSketch;
+    private final double medianAbsoluteDeviation;
+
+    public InternalMedianAbsoluteDeviation(String name,
+                                           List<PipelineAggregator> pipelineAggregators,
+                                           Map<String, Object> metaData,
+                                           DocValueFormat format,
+                                           TDigestState valuesSketch) {
+
+        super(name, pipelineAggregators, metaData);
+        this.format = Objects.requireNonNull(format);
+        this.valuesSketch = Objects.requireNonNull(valuesSketch);
+
+        this.medianAbsoluteDeviation = computeMedianAbsoluteDeviation(this.valuesSketch);
+    }
+
+    public InternalMedianAbsoluteDeviation(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        valuesSketch = TDigestState.read(in);
+        medianAbsoluteDeviation = in.readDouble();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        TDigestState.write(valuesSketch, out);
+        out.writeDouble(medianAbsoluteDeviation);
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        final TDigestState valueMerged = new TDigestState(valuesSketch.compression());
+        for (InternalAggregation aggregation : aggregations) {
+            final InternalMedianAbsoluteDeviation madAggregation = (InternalMedianAbsoluteDeviation) aggregation;
+            valueMerged.add(madAggregation.valuesSketch);
+        }
+
+        return new InternalMedianAbsoluteDeviation(name, pipelineAggregators(), metaData, format, valueMerged);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        final boolean anyResults = valuesSketch.size() > 0;
+        final Double mad = anyResults
+            ? getMedianAbsoluteDeviation()
+            : null;
+
+        builder.field(CommonFields.VALUE.getPreferredName(), mad);
+        if (format != DocValueFormat.RAW && anyResults) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(mad).toString());
+        }
+
+        return builder;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(valuesSketch);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalMedianAbsoluteDeviation other = (InternalMedianAbsoluteDeviation) obj;
+        return Objects.equals(valuesSketch, other.valuesSketch);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return MedianAbsoluteDeviationAggregationBuilder.NAME;
+    }
+
+    public TDigestState getValuesSketch() {
+        return valuesSketch;
+    }
+
+    @Override
+    public double value() {
+        return getMedianAbsoluteDeviation();
+    }
+
+    @Override
+    public double getMedianAbsoluteDeviation() {
+        return medianAbsoluteDeviation;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviation.java
new file mode 100644
index 0000000000000..7edc7682cea30
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviation.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+
+/**
+ * An aggregation that approximates the median absolute deviation of a numeric field
+ *
+ * @see <a href="https://en.wikipedia.org/wiki/Median_absolute_deviation">https://en.wikipedia.org/wiki/Median_absolute_deviation</a>
+ */
+public interface MedianAbsoluteDeviation extends NumericMetricsAggregation.SingleValue {
+
+    /**
+     * Returns the median absolute deviation statistic computed for this aggregation
+     */
+    double getMedianAbsoluteDeviation();
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregationBuilder.java
new file mode 100644
index 0000000000000..2f088ccff0dac
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregationBuilder.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class MedianAbsoluteDeviationAggregationBuilder extends LeafOnly<ValuesSource.Numeric, MedianAbsoluteDeviationAggregationBuilder> {
+
+    public static final String NAME = "median_absolute_deviation";
+
+    private static final ParseField COMPRESSION_FIELD = new ParseField("compression");
+
+    private static final ObjectParser<MedianAbsoluteDeviationAggregationBuilder, Void> PARSER;
+
+    static {
+        PARSER = new ObjectParser<>(NAME);
+        ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, false);
+        PARSER.declareDouble(MedianAbsoluteDeviationAggregationBuilder::compression, COMPRESSION_FIELD);
+    }
+
+    public static MedianAbsoluteDeviationAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new MedianAbsoluteDeviationAggregationBuilder(aggregationName), null);
+    }
+
+    private double compression = 1000d;
+
+    public MedianAbsoluteDeviationAggregationBuilder(String name) {
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+    }
+
+    public MedianAbsoluteDeviationAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        compression = in.readDouble();
+    }
+
+    protected MedianAbsoluteDeviationAggregationBuilder(MedianAbsoluteDeviationAggregationBuilder clone,
+                                                        AggregatorFactories.Builder factoriesBuilder,
+                                                        Map<String, Object> metaData) {
+        super(clone, factoriesBuilder, metaData);
+        this.compression = clone.compression;
+    }
+
+    /**
+     * Returns the compression factor of the t-digest sketches used
+     */
+    public double compression() {
+        return compression;
+    }
+
+    /**
+     * Set the compression factor of the t-digest sketches used
+     */
+    public MedianAbsoluteDeviationAggregationBuilder compression(double compression) {
+        if (compression <= 0d) {
+            throw new IllegalArgumentException(
+                "[" + COMPRESSION_FIELD.getPreferredName() + "] must be greater than 0. Found [" + compression + "] in [" + name + "]");
+        }
+        this.compression = compression;
+        return this;
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new MedianAbsoluteDeviationAggregationBuilder(this, factoriesBuilder, metaData);
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeDouble(compression);
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<ValuesSource.Numeric, ?> innerBuild(SearchContext context,
+                                                                                ValuesSourceConfig<ValuesSource.Numeric> config,
+                                                                                AggregatorFactory<?> parent,
+                                                                                AggregatorFactories.Builder subFactoriesBuilder)
+                                                                                throws IOException {
+
+        return new MedianAbsoluteDeviationAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData, compression);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(COMPRESSION_FIELD.getPreferredName(), compression);
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return Objects.hash(compression);
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        MedianAbsoluteDeviationAggregationBuilder other = (MedianAbsoluteDeviationAggregationBuilder) obj;
+        return Objects.equals(compression, other.compression);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregator.java
new file mode 100644
index 0000000000000..a9fc952c5d8ff
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregator.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.search.aggregations.metrics.mad.InternalMedianAbsoluteDeviation.computeMedianAbsoluteDeviation;
+
+public class MedianAbsoluteDeviationAggregator extends NumericMetricsAggregator.SingleValue {
+
+    private final ValuesSource.Numeric valuesSource;
+    private final DocValueFormat format;
+
+    private final double compression;
+
+    private ObjectArray<TDigestState> valueSketches;
+
+    MedianAbsoluteDeviationAggregator(String name,
+                                             SearchContext context,
+                                             Aggregator parent,
+                                             List<PipelineAggregator> pipelineAggregators,
+                                             Map<String, Object> metaData,
+                                             @Nullable ValuesSource.Numeric valuesSource,
+                                             DocValueFormat format,
+                                             double compression) throws IOException {
+
+        super(name, context, parent, pipelineAggregators, metaData);
+
+        this.valuesSource = valuesSource;
+        this.format = Objects.requireNonNull(format);
+        this.compression = compression;
+        this.valueSketches = context.bigArrays().newObjectArray(1);
+    }
+
+    private boolean hasDataForBucket(long bucketOrd) {
+        return bucketOrd < valueSketches.size() && valueSketches.get(bucketOrd) != null;
+    }
+
+    @Override
+    public double metric(long owningBucketOrd) {
+        if (hasDataForBucket(owningBucketOrd)) {
+            return computeMedianAbsoluteDeviation(valueSketches.get(owningBucketOrd));
+        } else {
+            return Double.NaN;
+        }
+    }
+
+    @Override
+    public boolean needsScores() {
+        return valuesSource != null && valuesSource.needsScores();
+    }
+
+    @Override
+    protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+        if (valuesSource == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+
+        final BigArrays bigArrays = context.bigArrays();
+        final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx);
+
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+
+                valueSketches = bigArrays.grow(valueSketches, bucket + 1);
+
+                TDigestState valueSketch = valueSketches.get(bucket);
+                if (valueSketch == null) {
+                    valueSketch = new TDigestState(compression);
+                    valueSketches.set(bucket, valueSketch);
+                }
+
+                if (values.advanceExact(doc)) {
+                    final int valueCount = values.docValueCount();
+                    for (int i = 0; i < valueCount; i++) {
+                        final double value = values.nextValue();
+                        valueSketch.add(value);
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) throws IOException {
+        if (hasDataForBucket(bucket)) {
+            final TDigestState valueSketch = valueSketches.get(bucket);
+            return new InternalMedianAbsoluteDeviation(name, pipelineAggregators(), metaData(), format, valueSketch);
+        } else {
+            return buildEmptyAggregation();
+        }
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new InternalMedianAbsoluteDeviation(name, pipelineAggregators(), metaData(), format, new TDigestState(compression));
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(valueSketches);
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregatorFactory.java
new file mode 100644
index 0000000000000..9c0ffcc9b2007
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/MedianAbsoluteDeviationAggregatorFactory.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class MedianAbsoluteDeviationAggregatorFactory extends
+        ValuesSourceAggregatorFactory<ValuesSource.Numeric, MedianAbsoluteDeviationAggregatorFactory> {
+
+    private final double compression;
+
+    MedianAbsoluteDeviationAggregatorFactory(String name,
+                                                    ValuesSourceConfig<ValuesSource.Numeric> config,
+                                                    SearchContext context,
+                                                    AggregatorFactory<?> parent,
+                                                    AggregatorFactories.Builder subFactoriesBuilder,
+                                                    Map<String, Object> metaData,
+                                                    double compression) throws IOException {
+
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
+        this.compression = compression;
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent,
+                                        List<PipelineAggregator> pipelineAggregators,
+                                        Map<String, Object> metaData) throws IOException {
+
+        return new MedianAbsoluteDeviationAggregator(
+            name,
+            context,
+            parent,
+            pipelineAggregators,
+            metaData,
+            null,
+            config.format(),
+            compression
+        );
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(ValuesSource.Numeric valuesSource,
+                                          Aggregator parent,
+                                          boolean collectsFromSingleBucket,
+                                          List<PipelineAggregator> pipelineAggregators,
+                                          Map<String, Object> metaData) throws IOException {
+
+        return new MedianAbsoluteDeviationAggregator(
+            name,
+            context,
+            parent,
+            pipelineAggregators,
+            metaData,
+            valuesSource,
+            config.format(),
+            compression
+        );
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/ParsedMedianAbsoluteDeviation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/ParsedMedianAbsoluteDeviation.java
new file mode 100644
index 0000000000000..21f8505faa9ab
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/mad/ParsedMedianAbsoluteDeviation.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.mad;
+
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.metrics.ParsedSingleValueNumericMetricsAggregation;
+
+import java.io.IOException;
+
+public class ParsedMedianAbsoluteDeviation extends ParsedSingleValueNumericMetricsAggregation implements MedianAbsoluteDeviation {
+
+    private static final ObjectParser<ParsedMedianAbsoluteDeviation, Void> PARSER = new ObjectParser<>(
+        ParsedMedianAbsoluteDeviation.class.getSimpleName(),
+        true,
+        ParsedMedianAbsoluteDeviation::new
+    );
+
+    static {
+        declareSingleValueFields(PARSER, Double.NaN);
+    }
+
+    public static ParsedMedianAbsoluteDeviation fromXContent(XContentParser parser, String name) {
+        ParsedMedianAbsoluteDeviation parsedMedianAbsoluteDeviation = PARSER.apply(parser, null);
+        parsedMedianAbsoluteDeviation.setName(name);
+        return parsedMedianAbsoluteDeviation;
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        final boolean hasValue = Double.isFinite(getMedianAbsoluteDeviation());
+        builder.field(CommonFields.VALUE.getPreferredName(), hasValue ? getMedianAbsoluteDeviation() : null);
+        if (hasValue && valueAsString != null) {
+            builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), valueAsString);
+        }
+        return builder;
+    }
+
+    @Override
+    public double getMedianAbsoluteDeviation() {
+        return value();
+    }
+
+    @Override
+    public String getType() {
+        return MedianAbsoluteDeviationAggregationBuilder.NAME;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index 4d911cb550b21..b671f95c446cb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -96,9 +96,7 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
             }
 
             // Add _aggs to params map for backwards compatibility (redundant with a context variable on the ReduceScript created below).
-            if (ScriptedMetricAggContexts.deprecatedAggParamEnabled()) {
-                params.put("_aggs", aggregationObjects);
-            }
+            params.put("_aggs", aggregationObjects);
 
             ScriptedMetricAggContexts.ReduceScript.Factory factory = reduceContext.scriptService().compile(
                 firstAggregation.reduceScript, ScriptedMetricAggContexts.ReduceScript.CONTEXT);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
index ca37b5618a2b9..876392dd3602a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
@@ -84,15 +84,13 @@ public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBu
         // When this is removed, aggState (as passed to ScriptedMetricAggregator) can be changed to Map<String, Object>, since
         // it won't be possible to completely replace it with another type as is possible when it's an entry in params.
         Object aggState = new HashMap<String, Object>();
-        if (ScriptedMetricAggContexts.deprecatedAggParamEnabled()) {
-            if (aggParams.containsKey("_agg") == false) {
-                // Add _agg if it wasn't added manually
-                aggParams.put("_agg", aggState);
-            } else {
-                // If it was added manually, also use it for the agg context variable to reduce the likelihood of
-                // weird behavior due to multiple different variables.
-                aggState = aggParams.get("_agg");
-            }
+        if (aggParams.containsKey("_agg") == false) {
+            // Add _agg if it wasn't added manually
+            aggParams.put("_agg", aggState);
+        } else {
+            // If it was added manually, also use it for the agg context variable to reduce the likelihood of
+            // weird behavior due to multiple different variables.
+            aggState = aggParams.get("_agg");
         }
 
         final ScriptedMetricAggContexts.InitScript initScript = this.initScript.newInstance(
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
index 11e46061d6786..b4cbd03116758 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightPhase.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.search.fetch.subphase.highlight;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -35,11 +33,10 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
+public class HighlightPhase implements FetchSubPhase {
     private final Map<String, Highlighter> highlighters;
 
-    public HighlightPhase(Settings settings, Map<String, Highlighter> highlighters) {
-        super(settings);
+    public HighlightPhase(Map<String, Highlighter> highlighters) {
         this.highlighters = highlighters;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 729466450ac8e..e8766685d9773 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
 import org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor;
 import org.elasticsearch.search.DocValueFormat;
@@ -78,10 +77,10 @@ public class QueryPhase implements SearchPhase {
     private final SuggestPhase suggestPhase;
     private RescorePhase rescorePhase;
 
-    public QueryPhase(Settings settings) {
+    public QueryPhase() {
         this.aggregationPhase = new AggregationPhase();
-        this.suggestPhase = new SuggestPhase(settings);
-        this.rescorePhase = new RescorePhase(settings);
+        this.suggestPhase = new SuggestPhase();
+        this.rescorePhase = new RescorePhase();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index 7baaa61bbb8c5..565fd8395dae6 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -22,8 +22,6 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -32,12 +30,7 @@
 /**
  * Rescore phase of a search request, used to run potentially expensive scoring models against the top matching documents.
  */
-public class RescorePhase extends AbstractComponent implements SearchPhase {
-
-    public RescorePhase(Settings settings) {
-        super(settings);
-    }
-
+public class RescorePhase implements SearchPhase {
     @Override
     public void preProcess(SearchContext context) {
     }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
index 874448b924c06..89b1f08958134 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
@@ -20,8 +20,6 @@
 
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
@@ -37,12 +35,7 @@
 /**
  * Suggest phase of a search request, used to collect suggestions
  */
-public class SuggestPhase extends AbstractComponent implements SearchPhase {
-
-    public SuggestPhase(Settings settings) {
-        super(settings);
-    }
-
+public class SuggestPhase implements SearchPhase {
     @Override
     public void preProcess(SearchContext context) {
     }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 791b59a1d5bb0..7e1873c3f0a6e 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -151,10 +151,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
     private final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor;
 
     @Inject
-    public RestoreService(Settings settings, ClusterService clusterService, RepositoriesService repositoriesService,
+    public RestoreService(ClusterService clusterService, RepositoriesService repositoriesService,
                           AllocationService allocationService, MetaDataCreateIndexService createIndexService,
                           MetaDataIndexUpgradeService metaDataIndexUpgradeService, ClusterSettings clusterSettings) {
-        super(settings);
         this.clusterService = clusterService;
         this.repositoriesService = repositoriesService;
         this.allocationService = allocationService;
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index bfe6d550ebd1b..b0af54b71a191 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -99,6 +99,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
     public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME_V6 = "internal:cluster/snapshot/update_snapshot";
     public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status";
 
+    private final Settings settings;
 
     private final ClusterService clusterService;
 
@@ -124,6 +125,7 @@ public SnapshotShardsService(Settings settings, ClusterService clusterService, S
                                  TransportService transportService, IndicesService indicesService,
                                  ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings);
+        this.settings = settings;
         this.indicesService = indicesService;
         this.snapshotsService = snapshotsService;
         this.transportService = transportService;
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 80427b197239d..3d0c14130373e 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -78,7 +78,6 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
     private final ByteSizeValue maxHeaderSize;
 
     public TaskManager(Settings settings, ThreadPool threadPool, Set<String> taskHeaders) {
-        super(settings);
         this.threadPool = threadPool;
         this.taskHeaders = new ArrayList<>(taskHeaders);
         this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
index 08d9d15f8135c..f500e9561d808 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
@@ -72,9 +72,8 @@ public class TaskResultsService extends AbstractComponent {
     private final TransportCreateIndexAction createIndexAction;
 
     @Inject
-    public TaskResultsService(Settings settings, Client client, ClusterService clusterService,
+    public TaskResultsService(Client client, ClusterService clusterService,
                               TransportCreateIndexAction createIndexAction) {
-        super(settings);
         this.client = client;
         this.clusterService = clusterService;
         this.createIndexAction = createIndexAction;
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 5ca27409f6cae..b9ebdf94dacef 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -161,8 +161,6 @@ public Collection<ExecutorBuilder> builders() {
         Setting.timeSetting("thread_pool.estimated_time_interval", TimeValue.timeValueMillis(200), Setting.Property.NodeScope);
 
     public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBuilders) {
-        super(settings);
-
         assert Node.NODE_NAME_SETTING.exists(settings);
 
         final Map<String, ExecutorBuilder> builders = new HashMap<>();
diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java
index 5f2635fac88d9..114bb8c986a07 100644
--- a/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java
+++ b/server/src/main/java/org/elasticsearch/transport/ConnectionManager.java
@@ -18,8 +18,8 @@
  */
 package org.elasticsearch.transport;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -67,16 +67,15 @@ public class ConnectionManager implements Closeable {
     private final DelegatingNodeConnectionListener connectionListener = new DelegatingNodeConnectionListener();
 
     public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool) {
-        this(settings, transport, threadPool, ConnectionProfile.buildDefaultConnectionProfile(settings));
+        this(settings, transport, threadPool, TcpTransport.PING_SCHEDULE.get(settings));
     }
 
-    public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool, ConnectionProfile defaultProfile) {
+    public ConnectionManager(Settings settings, Transport transport, ThreadPool threadPool, TimeValue pingSchedule) {
         this.transport = transport;
         this.threadPool = threadPool;
-        this.pingSchedule = TcpTransport.PING_SCHEDULE.get(settings);
-        this.defaultProfile = defaultProfile;
+        this.pingSchedule = pingSchedule;
+        this.defaultProfile = ConnectionProfile.buildDefaultConnectionProfile(settings);
         this.lifecycle.moveToStarted();
-
         if (pingSchedule.millis() > 0) {
             threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, new ScheduledPing());
         }
@@ -252,6 +251,10 @@ private void ensureOpen() {
         }
     }
 
+    TimeValue getPingSchedule() {
+        return pingSchedule;
+    }
+
     private class ScheduledPing extends AbstractLifecycleRunnable {
 
         private ScheduledPing() {
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
index a10fe550d7def..a7750920cfade 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
@@ -162,7 +162,7 @@ public String getKey(final String key) {
                     Setting.Property.NodeScope),
             REMOTE_CLUSTERS_SEEDS);
 
-
+    protected final Settings settings;
     protected final ClusterNameExpressionResolver clusterNameResolver;
 
     /**
@@ -170,8 +170,8 @@ public String getKey(final String key) {
      * @param settings the nodes level settings
      */
     protected RemoteClusterAware(Settings settings) {
-        super(settings);
-        this.clusterNameResolver = new ClusterNameExpressionResolver(settings);
+        this.settings = settings;
+        this.clusterNameResolver = new ClusterNameExpressionResolver();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
index d279d12ea0fdb..7060faefbe012 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.transport;
 
-import java.net.InetSocketAddress;
-import java.util.function.Supplier;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.SetOnce;
@@ -52,6 +50,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -69,6 +68,7 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 /**
@@ -98,6 +98,7 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo
     private volatile List<Supplier<DiscoveryNode>> seedNodes;
     private volatile boolean skipUnavailable;
     private final ConnectHandler connectHandler;
+    private final TimeValue initialConnectionTimeout;
     private SetOnce<ClusterName> remoteClusterName = new SetOnce<>();
 
     /**
@@ -109,18 +110,11 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo
      * @param connectionManager the connection manager to use for this remote connection
      * @param maxNumRemoteConnections the maximum number of connections to the remote cluster
      * @param nodePredicate a predicate to filter eligible remote nodes to connect to
+     * @param proxyAddress the proxy address
      */
     RemoteClusterConnection(Settings settings, String clusterAlias, List<Supplier<DiscoveryNode>> seedNodes,
             TransportService transportService, ConnectionManager connectionManager, int maxNumRemoteConnections,
-                            Predicate<DiscoveryNode> nodePredicate) {
-        this(settings, clusterAlias, seedNodes, transportService, connectionManager, maxNumRemoteConnections, nodePredicate, null);
-    }
-
-    RemoteClusterConnection(Settings settings, String clusterAlias, List<Supplier<DiscoveryNode>> seedNodes,
-            TransportService transportService, ConnectionManager connectionManager, int maxNumRemoteConnections, Predicate<DiscoveryNode>
-                                nodePredicate,
-                            String proxyAddress) {
-        super(settings);
+            Predicate<DiscoveryNode> nodePredicate, String proxyAddress) {
         this.transportService = transportService;
         this.maxNumRemoteConnections = maxNumRemoteConnections;
         this.nodePredicate = nodePredicate;
@@ -145,6 +139,7 @@ final class RemoteClusterConnection extends AbstractComponent implements Transpo
         // we register the transport service here as a listener to make sure we notify handlers on disconnect etc.
         connectionManager.addListener(transportService);
         this.proxyAddress = proxyAddress;
+        initialConnectionTimeout = RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
     }
 
     private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, DiscoveryNode node) {
@@ -155,7 +150,7 @@ private static DiscoveryNode maybeAddProxyAddress(String proxyAddress, Discovery
             InetSocketAddress proxyInetAddress = RemoteClusterAware.parseSeedAddress(proxyAddress);
             return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node
                 .getHostAddress(), new TransportAddress(proxyInetAddress), node.getAttributes(), node.getRoles(), node.getVersion());
-    }
+        }
     }
 
     /**
@@ -687,7 +682,7 @@ public void getConnectionInfo(ActionListener<RemoteConnectionInfo> listener) {
             // not connected we return immediately
             RemoteConnectionInfo remoteConnectionStats = new RemoteConnectionInfo(clusterAlias,
                 Collections.emptyList(), Collections.emptyList(), maxNumRemoteConnections, 0,
-                RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings), skipUnavailable);
+                initialConnectionTimeout, skipUnavailable);
             listener.onResponse(remoteConnectionStats);
         } else {
             NodesInfoRequest request = new NodesInfoRequest();
@@ -727,7 +722,7 @@ public void handleResponse(NodesInfoResponse response) {
                         seedNodes.stream().map(sup -> sup.get().getAddress()).collect(Collectors.toList()),
                         new ArrayList<>(httpAddresses),
                         maxNumRemoteConnections, connectedNodes.size(),
-                        RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings), skipUnavailable);
+                        initialConnectionTimeout, skipUnavailable);
                     listener.onResponse(remoteConnectionInfo);
                 }
 
@@ -747,7 +742,6 @@ public String executor() {
     RemoteConnectionInfo getLocalConnectionInfo() { // for tests
         List<TransportAddress> seedNodeAddresses = seedNodes.stream().map(node -> node.get().getAddress()).collect
             (Collectors.toList());
-        TimeValue initialConnectionTimeout = RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
         return new RemoteConnectionInfo(clusterAlias, Collections.emptyList(),
             seedNodeAddresses, maxNumRemoteConnections, connectedNodes.size(), initialConnectionTimeout, skipUnavailable);
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index 5ce2a6c955637..4e3bc40055d35 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -60,6 +60,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.settings.Setting.boolSetting;
+import static org.elasticsearch.common.settings.Setting.timeSetting;
 
 /**
  * Basic service for accessing remote clusters via gateway nodes
@@ -161,6 +162,12 @@ public String getKey(final String key) {
                             Setting.Property.NodeScope),
                     REMOTE_CLUSTERS_SEEDS);
 
+    public static final Setting.AffixSetting<TimeValue> REMOTE_CLUSTER_PING_SCHEDULE = Setting.affixKeySetting(
+            "cluster.remote.",
+            "transport.ping_schedule",
+            key -> timeSetting(key, TcpTransport.PING_SCHEDULE, Setting.Property.NodeScope),
+            REMOTE_CLUSTERS_SEEDS);
+
     private static final Predicate<DiscoveryNode> DEFAULT_NODE_PREDICATE = (node) -> Version.CURRENT.isCompatible(node.getVersion())
             && (node.isMasterNode() == false  || node.isDataNode() || node.isIngestNode());
 
@@ -206,10 +213,13 @@ private synchronized void updateRemoteClusters(Map<String, Tuple<String, List<Su
                 }
 
                 if (remote == null) { // this is a new cluster we have to add a new representation
-                    remote = new RemoteClusterConnection(settings, entry.getKey(), seedList, transportService,
-                            new ConnectionManager(settings, transportService.transport, transportService.threadPool), numRemoteConnections,
-                            getNodePredicate(settings), proxyAddress);
-                    remoteClusters.put(entry.getKey(), remote);
+                    String clusterAlias = entry.getKey();
+                    TimeValue pingSchedule = REMOTE_CLUSTER_PING_SCHEDULE.getConcreteSettingForNamespace(clusterAlias).get(settings);
+                    ConnectionManager connectionManager = new ConnectionManager(settings, transportService.transport,
+                        transportService.threadPool, pingSchedule);
+                    remote = new RemoteClusterConnection(settings, clusterAlias, seedList, transportService, connectionManager,
+                        numRemoteConnections, getNodePredicate(settings), proxyAddress);
+                    remoteClusters.put(clusterAlias, remote);
                 }
 
                 // now update the seed nodes no matter if it's new or already existing
@@ -335,31 +345,27 @@ public void onFailure(Exception e) {
      * @throws IllegalArgumentException if the remote cluster is unknown
      */
     public Transport.Connection getConnection(DiscoveryNode node, String cluster) {
-        RemoteClusterConnection connection = remoteClusters.get(cluster);
-        if (connection == null) {
-            throw new IllegalArgumentException("no such remote cluster: " + cluster);
-        }
-        return connection.getConnection(node);
+        return getRemoteClusterConnection(cluster).getConnection(node);
     }
 
     /**
      * Ensures that the given cluster alias is connected. If the cluster is connected this operation
      * will invoke the listener immediately.
      */
-    public void ensureConnected(String clusterAlias, ActionListener<Void> listener) {
-        RemoteClusterConnection remoteClusterConnection = remoteClusters.get(clusterAlias);
-        if (remoteClusterConnection == null) {
-            throw new IllegalArgumentException("no such remote cluster: " + clusterAlias);
-        }
-        remoteClusterConnection.ensureConnected(listener);
+    void ensureConnected(String clusterAlias, ActionListener<Void> listener) {
+        getRemoteClusterConnection(clusterAlias).ensureConnected(listener);
     }
 
     public Transport.Connection getConnection(String cluster) {
+        return getRemoteClusterConnection(cluster).getConnection();
+    }
+
+    RemoteClusterConnection getRemoteClusterConnection(String cluster) {
         RemoteClusterConnection connection = remoteClusters.get(cluster);
         if (connection == null) {
             throw new IllegalArgumentException("no such remote cluster: " + cluster);
         }
-        return connection.getConnection();
+        return connection;
     }
 
     @Override
@@ -381,7 +387,6 @@ synchronized void updateSkipUnavailable(String clusterAlias, Boolean skipUnavail
         }
     }
 
-
     @Override
     protected void updateRemoteCluster(String clusterAlias, List<String> addresses, String proxyAddress) {
         updateRemoteCluster(clusterAlias, addresses, proxyAddress, ActionListener.wrap((x) -> {}, (x) -> {}));
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 5abbbdb990237..f8ef3dd8d218d 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -117,8 +117,7 @@
 
 public abstract class TcpTransport extends AbstractLifecycleComponent implements Transport {
 
-    public static final String TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX = "transport_server_worker";
-    public static final String TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX = "transport_client_boss";
+    public static final String TRANSPORT_WORKER_THREAD_NAME_PREFIX = "transport_worker";
 
     public static final Setting<List<String>> HOST =
         listSetting("transport.host", emptyList(), Function.identity(), Setting.Property.NodeScope);
@@ -175,6 +174,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     public static final Setting<Settings> DEFAULT_FEATURES_SETTING = Setting.groupSetting(FEATURE_PREFIX + ".", Setting.Property.NodeScope);
     private final String[] features;
 
+    protected final Settings settings;
     private final CircuitBreakerService circuitBreakerService;
     protected final ThreadPool threadPool;
     private final BigArrays bigArrays;
@@ -213,6 +213,7 @@ public TcpTransport(String transportName, Settings settings, ThreadPool threadPo
                         CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
                         NetworkService networkService) {
         super(settings);
+        this.settings = settings;
         this.profileSettings = getProfileSettings(settings);
         this.threadPool = threadPool;
         this.bigArrays = bigArrays;
diff --git a/server/src/main/java/org/elasticsearch/transport/Transports.java b/server/src/main/java/org/elasticsearch/transport/Transports.java
index 26235ebb44cdb..d6968a855365a 100644
--- a/server/src/main/java/org/elasticsearch/transport/Transports.java
+++ b/server/src/main/java/org/elasticsearch/transport/Transports.java
@@ -29,9 +29,6 @@ public enum Transports {
     /** threads whose name is prefixed by this string will be considered network threads, even though they aren't */
     public static final String TEST_MOCK_TRANSPORT_THREAD_PREFIX = "__mock_network_thread";
 
-    public static final String NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX = "es_nio_transport_worker";
-    public static final String NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX = "es_nio_transport_acceptor";
-
     /**
      * Utility method to detect whether a thread is a network thread. Typically
      * used in assertions to make sure that we do not call blocking code from
@@ -41,11 +38,8 @@ public static boolean isTransportThread(Thread t) {
         final String threadName = t.getName();
         for (String s : Arrays.asList(
                 HttpServerTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX,
-                TcpTransport.TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX,
-                TcpTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX,
-                TEST_MOCK_TRANSPORT_THREAD_PREFIX,
-                NIO_TRANSPORT_WORKER_THREAD_NAME_PREFIX,
-                NIO_TRANSPORT_ACCEPTOR_THREAD_NAME_PREFIX)) {
+                TcpTransport.TRANSPORT_WORKER_THREAD_NAME_PREFIX,
+                TEST_MOCK_TRANSPORT_THREAD_PREFIX)) {
             if (threadName.contains(s)) {
                 return true;
             }
diff --git a/server/src/main/java/org/elasticsearch/usage/UsageService.java b/server/src/main/java/org/elasticsearch/usage/UsageService.java
index 9f742cca9d94a..9e1c2e0373452 100644
--- a/server/src/main/java/org/elasticsearch/usage/UsageService.java
+++ b/server/src/main/java/org/elasticsearch/usage/UsageService.java
@@ -40,9 +40,6 @@
 
 import org.elasticsearch.action.admin.cluster.node.usage.NodeUsage;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 
 import java.util.ArrayList;
@@ -53,14 +50,12 @@
 /**
  * A service to monitor usage of Elasticsearch features.
  */
-public class UsageService extends AbstractComponent {
+public class UsageService {
 
     private final List<BaseRestHandler> handlers;
     private final long sinceTime;
 
-    @Inject
-    public UsageService(Settings settings) {
-        super(settings);
+    public UsageService() {
         this.handlers = new ArrayList<>();
         this.sinceTime = System.currentTimeMillis();
     }
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index a766a3e3c5635..6a10dd854c7d9 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -111,8 +111,8 @@ public ActionResponse newResponse() {
 
     public void testSetupRestHandlerContainsKnownBuiltin() {
         SettingsModule settings = new SettingsModule(Settings.EMPTY);
-        UsageService usageService = new UsageService(settings.getSettings());
-        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+        UsageService usageService = new UsageService();
+        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(settings.getSettings()),
                 settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
                 null, usageService);
         actionModule.initRestHandlers(null);
@@ -134,8 +134,9 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         SettingsModule settings = new SettingsModule(Settings.EMPTY);
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
-            UsageService usageService = new UsageService(settings.getSettings());
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+            UsageService usageService = new UsageService();
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(),
+                    new IndexNameExpressionResolver(settings.getSettings()),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
                     singletonList(dupsMainAction), null, null, usageService);
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
@@ -166,9 +167,10 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         SettingsModule settings = new SettingsModule(Settings.EMPTY);
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
-            UsageService usageService = new UsageService(settings.getSettings());
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
-                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
+            UsageService usageService = new UsageService();
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(),
+                    new IndexNameExpressionResolver(settings.getSettings()), settings.getIndexScopedSettings(),
+                    settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
                     singletonList(registersFakeHandler), null, null, usageService);
             actionModule.initRestHandlers(null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
index 067dd0daea005..a75510cfb64ef 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -46,13 +45,13 @@
  */
 public class ClusterAllocationExplainActionTests extends ESTestCase {
 
-    private static final AllocationDeciders NOOP_DECIDERS = new AllocationDeciders(Settings.EMPTY, Collections.emptyList());
+    private static final AllocationDeciders NOOP_DECIDERS = new AllocationDeciders(Collections.emptyList());
 
     public void testInitializingOrRelocatingShardExplanation() throws Exception {
         ShardRoutingState shardRoutingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.RELOCATING);
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), shardRoutingState);
         ShardRouting shard = clusterState.getRoutingTable().index("idx").shard(0).primaryShard();
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.emptyList()),
             clusterState.getRoutingNodes(), clusterState, null, System.nanoTime());
         ClusterAllocationExplanation cae = TransportClusterAllocationExplainAction.explainShard(shard, allocation, null, randomBoolean(),
             new TestGatewayAllocator(), new ShardsAllocator() {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index 6e34a7510070e..853a991cf937f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -80,8 +80,8 @@ public void testSerializeRequest() throws IOException {
     }
 
     public void testClusterStateUpdateTask() {
-        AllocationService allocationService = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        AllocationService allocationService = new AllocationService(
+            new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         ClusterState clusterState = createInitialClusterState(allocationService);
         ClusterRerouteRequest req = new ClusterRerouteRequest();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
index 04ceefc9deabe..4c8ab2ea57787 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -81,6 +81,7 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30416")
     public void testCreateShrinkIndexToN() {
         assumeFalse("Broken on windows - https://github.com/elastic/elasticsearch/issues/33857", Constants.WINDOWS);
         int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
index d659e289349ac..2deaca7cd2765 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java
@@ -114,7 +114,7 @@ class TestTransportGetIndexAction extends TransportGetIndexAction {
         TestTransportGetIndexAction() {
             super(Settings.EMPTY, GetIndexActionTests.this.transportService, GetIndexActionTests.this.clusterService,
                 GetIndexActionTests.this.threadPool, settingsFilter, new ActionFilters(emptySet()),
-                new GetIndexActionTests.Resolver(Settings.EMPTY), indicesService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+                new GetIndexActionTests.Resolver(), indicesService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         }
 
         @Override
@@ -126,8 +126,8 @@ protected void doMasterOperation(GetIndexRequest request, String[] concreteIndic
     }
 
     static class Resolver extends IndexNameExpressionResolver {
-        Resolver(Settings settings) {
-            super(settings);
+        Resolver() {
+            super(Settings.EMPTY);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
index 85b85cf9e1469..7e4611e171b7f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
@@ -60,7 +60,7 @@ class TestTransportGetSettingsAction extends TransportGetSettingsAction {
         TestTransportGetSettingsAction() {
             super(Settings.EMPTY, GetSettingsActionTests.this.transportService, GetSettingsActionTests.this.clusterService,
                 GetSettingsActionTests.this.threadPool, settingsFilter, new ActionFilters(Collections.emptySet()),
-                new Resolver(Settings.EMPTY), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+                new Resolver(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         }
         @Override
         protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
@@ -129,8 +129,8 @@ public void testIncludeDefaultsWithFiltering() {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
-        Resolver(Settings settings) {
-            super(settings);
+        Resolver() {
+            super(Settings.EMPTY);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
index 454cfc8476b4d..acf624f095ced 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
@@ -104,8 +104,8 @@ public void testErrorCondition() {
         ClusterState clusterState = ClusterState.builder(createClusterState("source", randomIntBetween(2, 10), 0,
             Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
-        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        AllocationService service = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
@@ -127,8 +127,8 @@ public void testShrinkIndexSettings() {
                 .put("index.blocks.write", true)
                 .build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
-        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        AllocationService service = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index b3391c382ab49..148249a123082 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -183,8 +183,8 @@ private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistr
                 null,
                 xContentRegistry,
                 true);
-        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService,
-                new AliasValidator(Settings.EMPTY), null,
+        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(null, createIndexService,
+                new AliasValidator(), null,
                 new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry);
 
         final List<Throwable> throwables = new ArrayList<>();
@@ -217,7 +217,7 @@ private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
                 xContentRegistry(),
                 true);
         MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
-                Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService,
+                clusterService, createIndexService, new AliasValidator(), indicesService,
                 new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
 
         final List<Throwable> throwables = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 04ed2316492fb..dc1532b8301eb 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -167,7 +167,8 @@ public void testSimpleBulk9() throws Exception {
         BulkRequest bulkRequest = new BulkRequest();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
             () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
-        assertThat(exc.getMessage(), containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]"));
+        assertThat(exc.getMessage(),
+            containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]"));
     }
 
     public void testSimpleBulk10() throws Exception {
@@ -199,7 +200,8 @@ public void testBulkEmptyObject() throws Exception {
         BulkRequest bulkRequest = new BulkRequest();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
             () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
-        assertThat(exc.getMessage(), containsString("Malformed action/metadata line [" + emptyLine + "], expected FIELD_NAME but found [END_OBJECT]"));
+        assertThat(exc.getMessage(), containsString("Malformed action/metadata line ["
+            + emptyLine + "], expected FIELD_NAME but found [END_OBJECT]"));
     }
 
     // issue 7361
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java b/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
index f9cdfa092ae3c..decee8ceab714 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
@@ -230,7 +230,8 @@ private BulkItemResponse successfulResponse() {
         }
 
         private BulkItemResponse failedResponse() {
-            return new BulkItemResponse(1, OpType.INDEX, new BulkItemResponse.Failure("test", "test", "1", new EsRejectedExecutionException("pool full")));
+            return new BulkItemResponse(1, OpType.INDEX, new BulkItemResponse.Failure("test", "test", "1",
+                new EsRejectedExecutionException("pool full")));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
index a1abd4d61f7fd..36bb809478718 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
@@ -58,8 +58,8 @@ class TestTransportBulkAction extends TransportBulkAction {
 
         TestTransportBulkAction() {
             super(Settings.EMPTY, TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null,
-                    null, new ActionFilters(Collections.emptySet()), new Resolver(Settings.EMPTY),
-                    new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(Settings.EMPTY)));
+                    null, new ActionFilters(Collections.emptySet()), new Resolver(),
+                    new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()));
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index af4346baa949a..854b3876f562f 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -96,7 +96,7 @@ private TransportBulkAction createAction(boolean controlled, AtomicLong expected
             boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
-        IndexNameExpressionResolver resolver = new Resolver(Settings.EMPTY);
+        IndexNameExpressionResolver resolver = new Resolver();
         ActionFilters actionFilters = new ActionFilters(new HashSet<>());
 
         TransportCreateIndexAction createIndexAction = new TransportCreateIndexAction(
@@ -205,8 +205,8 @@ public void onFailure(Exception e) {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
-        Resolver(Settings settings) {
-            super(settings);
+        Resolver() {
+            super(Settings.EMPTY);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/action/delete/DeleteRequestTests.java b/server/src/test/java/org/elasticsearch/action/delete/DeleteRequestTests.java
new file mode 100644
index 0000000000000..5f897d0b8349b
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/delete/DeleteRequestTests.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.delete;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+
+public class DeleteRequestTests extends ESTestCase {
+
+    public void testValidation() {
+        {
+            final DeleteRequest request = new DeleteRequest("index4", "_doc", "0");
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, nullValue());
+        }
+
+        {
+            final DeleteRequest request = new DeleteRequest("index4", randomBoolean() ? "" : null, randomBoolean() ? "" : null);
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, not(nullValue()));
+            assertThat(validate.validationErrors(), hasItems("type is missing", "id is missing"));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java b/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
index be636e7d9875f..79c0765d7ac83 100644
--- a/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.explain;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -35,6 +36,10 @@
 import java.util.Collections;
 import java.util.List;
 
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+
 public class ExplainRequestTests extends ESTestCase {
     private NamedWriteableRegistry namedWriteableRegistry;
 
@@ -70,4 +75,24 @@ public void testSerialize() throws IOException {
             }
         }
     }
+
+    public void testValidation() {
+        {
+            final ExplainRequest request = new ExplainRequest("index4", "_doc", "0");
+            request.query(QueryBuilders.termQuery("field", "value"));
+
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, nullValue());
+        }
+
+        {
+            final ExplainRequest request = new ExplainRequest("index4", randomBoolean() ? "" : null, randomBoolean() ? "" : null);
+            request.query(QueryBuilders.termQuery("field", "value"));
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, not(nullValue()));
+            assertThat(validate.validationErrors(), hasItems("type is missing", "id is missing"));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java
new file mode 100644
index 0000000000000..fc2162e866240
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/get/GetRequestTests.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+
+public class GetRequestTests  extends ESTestCase {
+
+    public void testValidation() {
+        {
+            final GetRequest request = new GetRequest("index4", "_doc", "0");
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, nullValue());
+        }
+
+        {
+            final GetRequest request = new GetRequest("index4", randomBoolean() ? "" : null, randomBoolean() ? "" : null);
+            final ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, not(nullValue()));
+            assertThat(validate.validationErrors(), hasItems("type is missing", "id is missing"));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
index a130f10d65908..782edb517cb13 100644
--- a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
@@ -45,7 +45,8 @@ public void testSerialization() throws IOException {
         MultiGetShardRequest multiGetShardRequest = new MultiGetShardRequest(multiGetRequest, "index", 0);
         int numItems = iterations(10, 30);
         for (int i = 0; i < numItems; i++) {
-            MultiGetRequest.Item item = new MultiGetRequest.Item("alias-" + randomAlphaOfLength(randomIntBetween(1, 10)), "type", "id-" + i);
+            MultiGetRequest.Item item = new MultiGetRequest.Item("alias-" + randomAlphaOfLength(randomIntBetween(1, 10)),
+                "type", "id-" + i);
             if (randomBoolean()) {
                 int numFields = randomIntBetween(1, 5);
                 String[] fields = new String[numFields];
diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
index e302357809ea2..b7dc2be0fb722 100644
--- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java
@@ -21,12 +21,10 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
@@ -61,9 +59,7 @@ public void testFilterShards() throws InterruptedException {
         final boolean shard1 = randomBoolean();
         final boolean shard2 = randomBoolean();
 
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRequest request, SearchTask task,
                                      ActionListener<SearchService.CanMatchResponse> listener) {
@@ -75,7 +71,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRe
         AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
         GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("idx",
-            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
             2, randomBoolean(), primaryNode, replicaNode);
         final SearchRequest searchRequest = new SearchRequest();
         searchRequest.allowPartialSearchResults(true);
@@ -112,8 +108,7 @@ public void run() throws IOException {
     }
 
     public void testOldNodesTriggerException() {
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("search.remote.connect", false).build(), null, null);
+        SearchTransportService searchTransportService = new SearchTransportService(null, null);
         DiscoveryNode node = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), VersionUtils.randomVersionBetween(random(),
             VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_6_0)));
         SearchAsyncActionTests.MockConnection mockConnection = new SearchAsyncActionTests.MockConnection(node);
@@ -131,9 +126,7 @@ public void testFilterWithFailure() throws InterruptedException {
         lookup.put("node1", new SearchAsyncActionTests.MockConnection(primaryNode));
         lookup.put("node2", new SearchAsyncActionTests.MockConnection(replicaNode));
         final boolean shard1 = randomBoolean();
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRequest request, SearchTask task,
                                      ActionListener<SearchService.CanMatchResponse> listener) {
@@ -155,7 +148,7 @@ public void sendCanMatch(Transport.Connection connection, ShardSearchTransportRe
         AtomicReference<GroupShardsIterator<SearchShardIterator>> result = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
         GroupShardsIterator<SearchShardIterator> shardsIter = SearchAsyncActionTests.getShardsIter("idx",
-            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
             2, randomBoolean(), primaryNode, replicaNode);
 
         final SearchRequest searchRequest = new SearchRequest();
@@ -199,7 +192,7 @@ public void testLotsOfShards() throws InterruptedException {
 
 
         final SearchTransportService searchTransportService =
-                new SearchTransportService(Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+                new SearchTransportService(null, null) {
                     @Override
                     public void sendCanMatch(
                             Transport.Connection connection,
@@ -211,7 +204,7 @@ public void sendCanMatch(
                 };
 
         final CountDownLatch latch = new CountDownLatch(1);
-        final OriginalIndices originalIndices = new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed());
+        final OriginalIndices originalIndices = new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS);
         final GroupShardsIterator<SearchShardIterator> shardsIter =
                 SearchAsyncActionTests.getShardsIter("idx", originalIndices, 4096, randomBoolean(), primaryNode, replicaNode);
         final ExecutorService executor = Executors.newFixedThreadPool(randomIntBetween(1, Runtime.getRuntime().availableProcessors()));
diff --git a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
index bd1d6a85b09f5..55c39f735ce31 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ClearScrollControllerTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -70,7 +69,7 @@ public void onFailure(Exception e) {
             }
         };
         List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
-        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendClearAllScrollContexts(Transport.Connection connection, ActionListener<TransportResponse> listener) {
                 nodesInvoked.add(connection.getNode());
@@ -135,7 +134,7 @@ public void onFailure(Exception e) {
             }
         };
         List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
-        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
             public void sendFreeContext(Transport.Connection connection, long contextId,
@@ -213,7 +212,7 @@ public void onFailure(Exception e) {
             }
         };
         List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
-        SearchTransportService searchTransportService = new SearchTransportService(Settings.EMPTY, null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
 
             @Override
             public void sendFreeContext(Transport.Connection connection, long contextId,
diff --git a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
index c1bdf901a6d1b..aea46dd1dda51 100644
--- a/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/DfsQueryPhaseTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.MockDirectoryWrapper;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.Index;
@@ -57,11 +56,9 @@ public void testDfsWith2Shards() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
@@ -115,11 +112,9 @@ public void testDfsWith1ShardFailed() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
@@ -172,11 +167,9 @@ public void testFailPhaseOnException() throws IOException {
         results.get(0).termsStatistics(new Term[0], new TermStatistics[0]);
         results.get(1).termsStatistics(new Term[0], new TermStatistics[0]);
 
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteQuery(Transport.Connection connection, QuerySearchRequest request, SearchTask task,
                                          SearchActionListener<QuerySearchResult> listener) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
index 22950ab2ad01a..f19e51273c69f 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.document.DocumentField;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
@@ -69,9 +68,7 @@ public void testCollapseSingleHit() throws IOException {
                     .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum))
                         .collect(Collectors.toList()))));
             mockSearchPhaseContext.getRequest().source().query(originalQuery);
-            mockSearchPhaseContext.searchTransport = new SearchTransportService(
-                Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+            mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
                 @Override
                 void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
                     assertTrue(executedMultiSearch.compareAndSet(false, true));
@@ -143,9 +140,7 @@ public void testFailOneItemFailsEntirePhase() throws IOException {
         String collapseValue = randomBoolean() ? null : "boom";
         mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder()
             .collapse(new CollapseBuilder("someField").setInnerHits(new InnerHitBuilder().setName("foobarbaz"))));
-        mockSearchPhaseContext.searchTransport = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
                 assertTrue(executedMultiSearch.compareAndSet(false, true));
@@ -184,9 +179,7 @@ public void run() throws IOException {
 
     public void testSkipPhase() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
-        mockSearchPhaseContext.searchTransport = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
               fail("no collapsing here");
@@ -215,9 +208,7 @@ public void run() throws IOException {
 
     public void testSkipExpandCollapseNoHits() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
-        mockSearchPhaseContext.searchTransport = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
                 fail("expand should not try to send empty multi search request");
@@ -247,9 +238,7 @@ public void testExpandRequestOptions() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         boolean version = randomBoolean();
 
-        mockSearchPhaseContext.searchTransport = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
-
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
                 final QueryBuilder postFilter = QueryBuilders.existsQuery("foo");
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index e9795bfdf6f59..0c79138b8ef1a 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -21,7 +21,6 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.MockDirectoryWrapper;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.search.DocValueFormat;
@@ -45,7 +44,7 @@
 public class FetchSearchPhaseTests extends ESTestCase {
 
     public void testShortcutQueryAndFetchOptimization() throws IOException {
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
@@ -87,7 +86,7 @@ public void run() throws IOException {
 
     public void testFetchTwoDocument() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
@@ -105,8 +104,7 @@ public void testFetchTwoDocument() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -142,7 +140,7 @@ public void run() throws IOException {
 
     public void testFailFetchOneDoc() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
@@ -160,8 +158,7 @@ public void testFailFetchOneDoc() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -201,7 +198,7 @@ public void testFetchDocsConcurrently() throws IOException, InterruptedException
         int resultSetSize = randomIntBetween(0, 100);
         // we use at least 2 hits otherwise this is subject to single shard optimization and we trip an assert...
         int numHits = randomIntBetween(2, 100); // also numshards --> 1 hit per shard
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(numHits);
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
@@ -214,8 +211,7 @@ public void testFetchDocsConcurrently() throws IOException, InterruptedException
             queryResult.setShardIndex(i);
             results.consumeResult(queryResult);
         }
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -258,7 +254,7 @@ public void run() throws IOException {
 
     public void testExceptionFailsPhase() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
@@ -276,8 +272,7 @@ public void testExceptionFailsPhase() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
         AtomicInteger numFetches = new AtomicInteger(0);
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -312,7 +307,7 @@ public void run() throws IOException {
 
     public void testCleanupIrrelevantContexts() throws IOException { // contexts that are not fetched should be cleaned up
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(2);
-        SearchPhaseController controller = new SearchPhaseController(Settings.EMPTY,
+        SearchPhaseController controller = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
@@ -330,8 +325,7 @@ public void testCleanupIrrelevantContexts() throws IOException { // contexts tha
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(
-            Settings.builder().put("cluster.remote.connect", false).build(), null, null) {
+        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 4e536cc14d61b..5d56576d91326 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -58,27 +58,27 @@ public void testSimpleAdd() throws Exception {
         assertThat(request.requests().get(0).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
-                equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+                equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
         assertThat(request.requests().get(0).types().length,
                 equalTo(0));
         assertThat(request.requests().get(1).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(1).indicesOptions(),
-                equalTo(IndicesOptions.fromOptions(false, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+                equalTo(IndicesOptions.fromOptions(false, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
         assertThat(request.requests().get(1).types()[0],
                 equalTo("type1"));
         assertThat(request.requests().get(2).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(2).indicesOptions(),
-                equalTo(IndicesOptions.fromOptions(false, true, true, false, IndicesOptions.strictExpandOpenAndForbidClosed())));
+                equalTo(IndicesOptions.fromOptions(false, true, true, false, SearchRequest.DEFAULT_INDICES_OPTIONS)));
         assertThat(request.requests().get(3).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(3).indicesOptions(),
-                equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+                equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
         assertThat(request.requests().get(4).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(4).indicesOptions(),
-                equalTo(IndicesOptions.fromOptions(true, false, false, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+                equalTo(IndicesOptions.fromOptions(true, false, false, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
 
         assertThat(request.requests().get(5).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(5).types().length, equalTo(0));
@@ -98,7 +98,7 @@ public void testSimpleAddWithCarriageReturn() throws Exception {
         assertThat(request.requests().size(), equalTo(1));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
-            equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
+            equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
         assertThat(request.requests().get(0).types().length, equalTo(0));
     }
 
@@ -273,11 +273,11 @@ private static MultiSearchRequest createMultiSearchRequest() throws IOException
 
             // only expand_wildcards, ignore_unavailable and allow_no_indices can be specified from msearch api, so unset other options:
             IndicesOptions randomlyGenerated = searchRequest.indicesOptions();
-            IndicesOptions msearchDefault = IndicesOptions.strictExpandOpenAndForbidClosed();
+            IndicesOptions msearchDefault = SearchRequest.DEFAULT_INDICES_OPTIONS;
             searchRequest.indicesOptions(IndicesOptions.fromOptions(
                     randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(),
                     randomlyGenerated.expandWildcardsClosed(), msearchDefault.allowAliasesToMultipleIndices(),
-                    msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases()
+                    msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), msearchDefault.ignoreThrottled()
             ));
 
             request.add(searchRequest);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index b72f2686c57d3..7075cd9fb903e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -30,7 +29,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -83,7 +81,7 @@ public void onFailure(Exception e) {
 
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
-            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
             10, randomBoolean(), primaryNode, replicaNode);
         int numSkipped = 0;
         for (SearchShardIterator iter : shardsIter) {
@@ -93,7 +91,7 @@ public void onFailure(Exception e) {
             }
         }
 
-        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null, null);
+        SearchTransportService transportService = new SearchTransportService(null, null);
         Map<String, Transport.Connection> lookup = new HashMap<>();
         Map<ShardId, Boolean> seenShard = new ConcurrentHashMap<>();
         lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
@@ -182,9 +180,9 @@ public void onFailure(Exception e) {
 
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
-            new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
             10, randomBoolean(), primaryNode, replicaNode);
-        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null, null);
+        SearchTransportService transportService = new SearchTransportService(null, null);
         Map<String, Transport.Connection> lookup = new HashMap<>();
         Map<ShardId, Boolean> seenShard = new ConcurrentHashMap<>();
         lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
@@ -282,10 +280,10 @@ public void onFailure(Exception e) {
         Map<DiscoveryNode, Set<Long>> nodeToContextMap = newConcurrentMap();
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
-                new OriginalIndices(new String[]{"idx"}, IndicesOptions.strictExpandOpenAndForbidClosed()),
+                new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
                 randomIntBetween(1, 10), randomBoolean(), primaryNode, replicaNode);
         AtomicInteger numFreedContext = new AtomicInteger();
-        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null, null) {
+        SearchTransportService transportService = new SearchTransportService(null, null) {
             @Override
             public void sendFreeContext(Transport.Connection connection, long contextId, OriginalIndices originalIndices) {
                 numFreedContext.incrementAndGet();
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
index 393c45fa57242..b6a6b8786b516 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java
@@ -22,7 +22,6 @@
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
@@ -67,7 +66,7 @@ public class SearchPhaseControllerTests extends ESTestCase {
 
     @Before
     public void setup() {
-        searchPhaseController = new SearchPhaseController(Settings.EMPTY,
+        searchPhaseController = new SearchPhaseController(
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index e529af97c800d..3e4747a4db757 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -80,7 +80,7 @@ public void testMergeShardsIterators() {
         GroupShardsIterator<ShardIterator> localShardsIterator = new GroupShardsIterator<>(localShardIterators);
 
         OriginalIndices localIndices = new OriginalIndices(new String[]{"local_alias", "local_index_2"},
-                IndicesOptions.strictExpandOpenAndForbidClosed());
+                SearchRequest.DEFAULT_INDICES_OPTIONS);
 
         OriginalIndices remoteIndices = new OriginalIndices(new String[]{"remote_alias", "remote_index_2"},
                 IndicesOptions.strictExpandOpen());
@@ -185,9 +185,9 @@ public void testProcessRemoteShards() {
 
             Map<String, OriginalIndices> remoteIndicesByCluster = new HashMap<>();
             remoteIndicesByCluster.put("test_cluster_1",
-                new OriginalIndices(new String[]{"fo*", "ba*"}, IndicesOptions.strictExpandOpenAndForbidClosed()));
+                new OriginalIndices(new String[]{"fo*", "ba*"}, SearchRequest.DEFAULT_INDICES_OPTIONS));
             remoteIndicesByCluster.put("test_cluster_2",
-                new OriginalIndices(new String[]{"x*"}, IndicesOptions.strictExpandOpenAndForbidClosed()));
+                new OriginalIndices(new String[]{"x*"}, SearchRequest.DEFAULT_INDICES_OPTIONS));
             Map<String, AliasFilter> remoteAliases = new HashMap<>();
             TransportSearchAction.processRemoteShards(searchShardsResponseMap, remoteIndicesByCluster, iteratorList,
                 remoteAliases);
@@ -274,6 +274,6 @@ private static OriginalIndices randomOriginalIndices() {
             localIndices[i] = randomAlphaOfLengthBetween(3, 10);
         }
         return new OriginalIndices(localIndices, IndicesOptions.fromOptions(randomBoolean(),
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index e36fe90ea92bb..f7fd39a038b58 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -51,9 +51,10 @@ public class IndicesOptionsTests extends ESTestCase {
     public void testSerialization() throws Exception {
         int iterations = randomIntBetween(5, 20);
         for (int i = 0; i < iterations; i++) {
-            Version version = randomVersionBetween(random(), Version.V_6_4_0, null);
+            Version version = randomVersionBetween(random(), Version.V_6_6_0, null);
             IndicesOptions indicesOptions = IndicesOptions.fromOptions(
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                randomBoolean());
 
             BytesStreamOutput output = new BytesStreamOutput();
             output.setVersion(version);
@@ -78,9 +79,9 @@ public void testSerialization() throws Exception {
     public void testSerializationPre70() throws Exception {
         int iterations = randomIntBetween(5, 20);
         for (int i = 0; i < iterations; i++) {
-            Version version = randomVersionBetween(random(), null, Version.V_6_4_0);
+            Version version = randomVersionBetween(random(), null, Version.V_6_6_0);
             IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-                    randomBoolean(), randomBoolean(), randomBoolean());
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
             BytesStreamOutput output = new BytesStreamOutput();
             output.setVersion(version);
@@ -103,6 +104,12 @@ public void testSerializationPre70() throws Exception {
             } else {
                 assertFalse(indicesOptions2.ignoreAliases());
             }
+            if (output.getVersion().onOrAfter(Version.V_6_6_0)) {
+                assertEquals(indicesOptions2.ignoreThrottled(), indicesOptions.ignoreThrottled());
+            } else {
+                assertFalse(indicesOptions2.ignoreThrottled()); // make sure we never write this option to pre 6.6
+            }
+
         }
     }
 
@@ -114,9 +121,10 @@ public void testFromOptions() {
         boolean allowAliasesToMultipleIndices = randomBoolean();
         boolean forbidClosedIndices = randomBoolean();
         boolean ignoreAliases = randomBoolean();
+        boolean ingoreThrottled = randomBoolean();
 
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices,expandToOpenIndices,
-                expandToClosedIndices, allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases);
+                expandToClosedIndices, allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases, ingoreThrottled);
 
         assertThat(indicesOptions.ignoreUnavailable(), equalTo(ignoreUnavailable));
         assertThat(indicesOptions.allowNoIndices(), equalTo(allowNoIndices));
@@ -126,6 +134,7 @@ public void testFromOptions() {
         assertThat(indicesOptions.allowAliasesToMultipleIndices(), equalTo(allowAliasesToMultipleIndices));
         assertThat(indicesOptions.forbidClosedIndices(), equalTo(forbidClosedIndices));
         assertEquals(ignoreAliases, indicesOptions.ignoreAliases());
+        assertEquals(ingoreThrottled, indicesOptions.ignoreThrottled());
     }
 
     public void testFromOptionsWithDefaultOptions() {
@@ -135,7 +144,7 @@ public void testFromOptionsWithDefaultOptions() {
         boolean expandToClosedIndices = randomBoolean();
 
         IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices,expandToOpenIndices,
                 expandToClosedIndices, defaultOptions);
@@ -168,14 +177,16 @@ public void testFromParameters() {
         }
         boolean ignoreUnavailable = randomBoolean();
         String ignoreUnavailableString = Boolean.toString(ignoreUnavailable);
+        boolean ignoreThrottled = randomBoolean();
+        String ignoreThrottledString = Boolean.toString(ignoreThrottled);
         boolean allowNoIndices = randomBoolean();
         String allowNoIndicesString = Boolean.toString(allowNoIndices);
 
         IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-                randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
         IndicesOptions updatedOptions = IndicesOptions.fromParameters(expandWildcardsString, ignoreUnavailableString,
-                allowNoIndicesString, defaultOptions);
+                allowNoIndicesString, ignoreThrottled, defaultOptions);
 
         assertEquals(expandWildcardsOpen, updatedOptions.expandWildcardsOpen());
         assertEquals(expandWildcardsClosed, updatedOptions.expandWildcardsClosed());
@@ -189,19 +200,19 @@ public void testFromParameters() {
     public void testSimpleByteBWC() {
         Map<Byte, IndicesOptions> old = new HashMap<>();
         // These correspond to each individual option (bit) in the old byte-based IndicesOptions
-        old.put((byte) 0, IndicesOptions.fromOptions(false, false, false, false, true, false, false));
-        old.put((byte) 1, IndicesOptions.fromOptions(true, false, false, false, true, false, false));
-        old.put((byte) 2, IndicesOptions.fromOptions(false, true, false, false, true, false, false));
-        old.put((byte) 4, IndicesOptions.fromOptions(false, false, true, false, true, false, false));
-        old.put((byte) 8, IndicesOptions.fromOptions(false, false, false, true, true, false, false));
-        old.put((byte) 16, IndicesOptions.fromOptions(false, false, false, false, false, false, false));
-        old.put((byte) 32, IndicesOptions.fromOptions(false, false, false, false, true, true, false));
-        old.put((byte) 64, IndicesOptions.fromOptions(false, false, false, false, true, false, true));
+        old.put((byte) 0, IndicesOptions.fromOptions(false, false, false, false, true, false, false, false));
+        old.put((byte) 1, IndicesOptions.fromOptions(true, false, false, false, true, false, false, false));
+        old.put((byte) 2, IndicesOptions.fromOptions(false, true, false, false, true, false, false, false));
+        old.put((byte) 4, IndicesOptions.fromOptions(false, false, true, false, true, false, false, false));
+        old.put((byte) 8, IndicesOptions.fromOptions(false, false, false, true, true, false, false, false));
+        old.put((byte) 16, IndicesOptions.fromOptions(false, false, false, false, false, false, false, false));
+        old.put((byte) 32, IndicesOptions.fromOptions(false, false, false, false, true, true, false, false));
+        old.put((byte) 64, IndicesOptions.fromOptions(false, false, false, false, true, false, true, false));
         // Test a few multi-selected options
-        old.put((byte) 13, IndicesOptions.fromOptions(true, false, true, true, true, false, false));
-        old.put((byte) 19, IndicesOptions.fromOptions(true, true, false, false, false, false, false));
-        old.put((byte) 24, IndicesOptions.fromOptions(false, false, false, true, false, false, false));
-        old.put((byte) 123, IndicesOptions.fromOptions(true, true, false, true, false, true, true));
+        old.put((byte) 13, IndicesOptions.fromOptions(true, false, true, true, true, false, false, false));
+        old.put((byte) 19, IndicesOptions.fromOptions(true, true, false, false, false, false, false, false));
+        old.put((byte) 24, IndicesOptions.fromOptions(false, false, false, true, false, false, false, false));
+        old.put((byte) 123, IndicesOptions.fromOptions(true, true, false, true, false, true, true, false));
 
         for (Map.Entry<Byte, IndicesOptions> entry : old.entrySet()) {
             IndicesOptions indicesOptions2 = IndicesOptions.fromByte(entry.getKey());
@@ -213,11 +224,13 @@ public void testSimpleByteBWC() {
 
     public void testEqualityAndHashCode() {
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(
-            randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+            randomBoolean());
 
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(indicesOptions, opts -> {
             return IndicesOptions.fromOptions(opts.ignoreUnavailable(), opts.allowNoIndices(), opts.expandWildcardsOpen(),
-                opts.expandWildcardsClosed(), opts.allowAliasesToMultipleIndices(), opts.forbidClosedIndices(), opts.ignoreAliases());
+                opts.expandWildcardsClosed(), opts.allowAliasesToMultipleIndices(), opts.forbidClosedIndices(), opts.ignoreAliases(),
+                opts.ignoreThrottled());
         }, opts -> {
             boolean mutated = false;
             boolean ignoreUnavailable = opts.ignoreUnavailable();
@@ -227,6 +240,7 @@ public void testEqualityAndHashCode() {
             boolean allowAliasesToMulti = opts.allowAliasesToMultipleIndices();
             boolean forbidClosed = opts.forbidClosedIndices();
             boolean ignoreAliases = opts.ignoreAliases();
+            boolean ignoreThrottled = opts.ignoreThrottled();
             while (mutated == false) {
                 if (randomBoolean()) {
                     ignoreUnavailable = !ignoreUnavailable;
@@ -256,9 +270,13 @@ public void testEqualityAndHashCode() {
                     ignoreAliases = !ignoreAliases;
                     mutated = true;
                 }
+                if (randomBoolean()) {
+                    ignoreThrottled = !ignoreThrottled;
+                    mutated = true;
+                }
             }
             return IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed,
-                allowAliasesToMulti, forbidClosed, ignoreAliases);
+                allowAliasesToMulti, forbidClosed, ignoreAliases, ignoreThrottled);
         });
     }
 
@@ -268,6 +286,7 @@ public void testFromMap() {
                 null : randomSubsetOf(Arrays.asList("open", "closed"));
         Boolean ignoreUnavailable = randomBoolean() ? null : randomBoolean();
         Boolean allowNoIndices = randomBoolean() ? null : randomBoolean();
+        Boolean ignoreThrottled = randomBoolean() ? null : randomBoolean();
 
         Map<String, Object> settings = new HashMap<>();
 
@@ -283,6 +302,10 @@ public void testFromMap() {
             settings.put("allow_no_indices", allowNoIndices);
         }
 
+        if (ignoreThrottled != null) {
+            settings.put("ignore_throttled", ignoreThrottled);
+        }
+
         IndicesOptions fromMap = IndicesOptions.fromMap(settings, defaults);
 
         boolean open = wildcardStates != null ? wildcardStates.contains("open") : defaults.expandWildcardsOpen();
@@ -292,6 +315,7 @@ public void testFromMap() {
 
         assertEquals(fromMap.ignoreUnavailable(), ignoreUnavailable == null ? defaults.ignoreUnavailable() : ignoreUnavailable);
         assertEquals(fromMap.allowNoIndices(), allowNoIndices == null ? defaults.allowNoIndices() : allowNoIndices);
+        assertEquals(fromMap.ignoreThrottled(), ignoreThrottled == null ? defaults.ignoreThrottled() : ignoreThrottled);
     }
 
     public void testToXContent() throws IOException {
@@ -324,5 +348,6 @@ public void testToXContent() throws IOException {
         }
         assertEquals(map.get("ignore_unavailable"), options.contains(Option.IGNORE_UNAVAILABLE));
         assertEquals(map.get("allow_no_indices"), options.contains(Option.ALLOW_NO_INDICES));
+        assertEquals(map.get("ignore_throttled"), options.contains(Option.IGNORE_THROTTLED));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 9bed5e7901704..21f3355030c69 100644
--- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -172,8 +172,8 @@ public Map<ShardRouting, Object> getResults() {
 
     class MyResolver extends IndexNameExpressionResolver {
         MyResolver() {
-            super(Settings.EMPTY);
-        }
+           super(Settings.EMPTY);
+       }            
 
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index f592b3d803af5..72f14c99542e2 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -132,7 +132,7 @@ class Response extends ActionResponse {}
     class Action extends TransportMasterNodeAction<Request, Response> {
         Action(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, ThreadPool threadPool) {
             super(settings, actionName, transportService, clusterService, threadPool,
-                    new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY), Request::new);
+                    new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(settings), Request::new);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 4f9da5b9dbc94..4e8f7cb3e4d1a 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -167,7 +167,7 @@ public void setUp() throws Exception {
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
-        shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
+        shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
         action = new TestAction(Settings.EMPTY, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
     }
 
@@ -1120,7 +1120,7 @@ private class TestAction extends TransportReplicationAction<Request, Request, Te
                    ThreadPool threadPool) {
             super(settings, actionName, transportService, clusterService, mockIndicesService(clusterService), threadPool,
                 shardStateAction,
-                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
+                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(settings),
                 Request::new, Request::new, ThreadPool.Names.SAME);
             this.withDocumentFailureOnPrimary = false;
             this.withDocumentFailureOnReplica = false;
@@ -1131,7 +1131,7 @@ private class TestAction extends TransportReplicationAction<Request, Request, Te
                    ThreadPool threadPool, boolean withDocumentFailureOnPrimary, boolean withDocumentFailureOnReplica) {
             super(settings, actionName, transportService, clusterService, mockIndicesService(clusterService), threadPool,
                 shardStateAction,
-                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
+                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(settings),
                 Request::new, Request::new, ThreadPool.Names.SAME);
             this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary;
             this.withDocumentFailureOnReplica = withDocumentFailureOnReplica;
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index 18bbe5b9593de..62cd9765562ab 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -259,7 +259,7 @@ public void testReplicaProxy() throws InterruptedException, ExecutionException {
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
-        ShardStateAction shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
+        ShardStateAction shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
         TestAction action = new TestAction(Settings.EMPTY, "internal:testAction", transportService,
                 clusterService, shardStateAction, threadPool);
         final String index = "test";
diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
index e2ae7a0d3ae8f..ca67e44a499a7 100644
--- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.update;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -61,10 +61,14 @@
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.elasticsearch.script.MockScriptEngine.mockInlineScript;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class UpdateRequestTests extends ESTestCase {
@@ -135,9 +139,7 @@ public void setUp() throws Exception {
         final MockScriptEngine engine = new MockScriptEngine("mock", scripts, Collections.emptyMap());
         Map<String, ScriptEngine> engines = Collections.singletonMap(engine.getType(), engine);
         ScriptService scriptService = new ScriptService(baseSettings, engines, ScriptModule.CORE_CONTEXTS);
-        final Settings settings = settings(Version.CURRENT).build();
-
-        updateHelper = new UpdateHelper(settings, scriptService);
+        updateHelper = new UpdateHelper(scriptService);
     }
 
     public void testFromXContent() throws Exception {
@@ -278,17 +280,62 @@ public void testFromXContent() throws Exception {
         assertThat(((Map) doc.get("compound")).get("field2").toString(), equalTo("value2"));
     }
 
+    public void testFromXContentBothScriptAndNestedDocs() throws Exception {
+        // related to https://github.com/elastic/elasticsearch/issues/34069
+        UpdateRequest request = new UpdateRequest("test", "type1", "1")
+            .fromXContent(
+                createParser(JsonXContent.jsonXContent,
+                    new BytesArray("{\"any_odd_name_for_update\":{\"doc\":{\"message\":\"set by update:doc\"}},"
+                        + "\"script\":{\"source\":\"ctx._source.message = 'set by script'\"}}")));
+
+        assertWarnings("Unknown field [any_odd_name_for_update] used in UpdateRequest"
+            + " which has no value and will not be accepted in future");
+
+        assertThat(request.doc(), notNullValue());
+        assertThat(request.script(), notNullValue());
+
+        ActionRequestValidationException validate = request.validate();
+        assertThat(validate, notNullValue());
+        assertThat(validate.validationErrors(), not(empty()));
+        assertThat(String.valueOf(validate.validationErrors()),
+            validate.validationErrors(), contains("can't provide both script and doc"));
+
+        request = new UpdateRequest("test", "type1", "1")
+            .fromXContent(
+                createParser(JsonXContent.jsonXContent,
+                    new BytesArray("{\"whatever\": {\"script\":{\"source\":\"ctx._source.message = 'set by script'\"}},"
+                        + "\"nested_update1\":{\"nested_update2\":{\"doc\":{\"message\":\"set by update:doc\"}}}}")));
+
+        assertWarnings("Unknown field [whatever] used in UpdateRequest"
+            + " which has no value and will not be accepted in future",
+            "Unknown field [nested_update1] used in UpdateRequest"
+                + " which has no value and will not be accepted in future",
+            "Unknown field [nested_update2] used in UpdateRequest"
+                + " which has no value and will not be accepted in future");
+
+        assertThat(request.doc(), notNullValue());
+        assertThat(request.script(), notNullValue());
+
+        validate = request.validate();
+        assertThat(validate, notNullValue());
+        assertThat(validate.validationErrors(), not(empty()));
+        assertThat(String.valueOf(validate.validationErrors()),
+            validate.validationErrors(), contains("can't provide both script and doc"));
+    }
+
     // Related to issue 15338
     public void testFieldsParsing() throws Exception {
         UpdateRequest request = new UpdateRequest("test", "type1", "1").fromXContent(
                 createParser(JsonXContent.jsonXContent, new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}")));
         assertThat(request.doc().sourceAsMap().get("field1").toString(), equalTo("value1"));
         assertThat(request.fields(), arrayContaining("_source"));
+        assertWarnings("Deprecated field [fields] used, expected [_source] instead");
 
         request = new UpdateRequest("test", "type2", "2").fromXContent(createParser(JsonXContent.jsonXContent,
                 new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}")));
         assertThat(request.doc().sourceAsMap().get("field2").toString(), equalTo("value2"));
         assertThat(request.fields(), arrayContaining("field1", "field2"));
+        assertWarnings("Deprecated field [fields] used, expected [_source] instead");
     }
 
     public void testFetchSourceParsing() throws Exception {
@@ -445,7 +492,9 @@ public void testToAndFromXContent() throws IOException {
             BytesReference source = RandomObjects.randomSource(random(), xContentType);
             updateRequest.upsert(new IndexRequest().source(source, xContentType));
         }
-        if (randomBoolean()) {
+
+        final boolean fieldsAreUsed = randomBoolean();
+        if (fieldsAreUsed) {
             String[] fields = new String[randomIntBetween(0, 5)];
             for (int i = 0; i < fields.length; i++) {
                 fields[i] = randomAlphaOfLength(5);
@@ -496,6 +545,10 @@ public void testToAndFromXContent() throws IOException {
 
         BytesReference finalBytes = toXContent(parsedUpdateRequest, xContentType, humanReadable);
         assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+
+        if (fieldsAreUsed) {
+            assertWarnings("Deprecated field [fields] used, expected [_source] instead");
+        }
     }
 
     public void testToValidateUpsertRequestAndVersion() {
@@ -513,6 +566,25 @@ public void testToValidateUpsertRequestWithVersion() {
         assertThat(updateRequest.validate().validationErrors(), contains("can't provide version in upsert request"));
     }
 
+    public void testValidate() {
+        {
+            UpdateRequest request = new UpdateRequest("index", "type", "id");
+            request.doc("{}", XContentType.JSON);
+            ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, nullValue());
+        }
+
+        {
+            UpdateRequest request = new UpdateRequest("index", randomBoolean() ? "" : null, randomBoolean() ? "" : null);
+            request.doc("{}", XContentType.JSON);
+            ActionRequestValidationException validate = request.validate();
+
+            assertThat(validate, not(nullValue()));
+            assertThat(validate.validationErrors(), hasItems("type is missing", "id is missing"));
+        }
+    }
+
     public void testParentAndRoutingExtraction() throws Exception {
         GetResult getResult = new GetResult("test", "type", "1", 0, false, null, null);
         IndexRequest indexRequest = new IndexRequest("test", "type", "1");
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 61507a8363d95..ae1caa787d4be 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -74,8 +74,8 @@ public static class TestPlugin extends Plugin implements ActionPlugin {
 
         private final BlockingActionFilter blockingActionFilter;
 
-        public TestPlugin(Settings settings) {
-            blockingActionFilter = new BlockingActionFilter(settings);
+        public TestPlugin() {
+            blockingActionFilter = new BlockingActionFilter();
         }
 
         @Override
@@ -87,10 +87,6 @@ public List<ActionFilter> getActionFilters() {
     public static class BlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple {
         private Set<String> blockedActions = emptySet();
 
-        public BlockingActionFilter(Settings settings) {
-            super(settings);
-        }
-
         @Override
         protected boolean apply(String action, ActionRequest request, ActionListener<?> listener) {
             if (blockedActions.contains(action)) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index efd8026645249..c479bc8122412 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -65,8 +65,7 @@ public class ClusterModuleTests extends ModuleTestCase {
     private ClusterService clusterService = new ClusterService(Settings.EMPTY,
         new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, Collections.emptyMap());
     static class FakeAllocationDecider extends AllocationDecider {
-        protected FakeAllocationDecider(Settings settings) {
-            super(settings);
+        protected FakeAllocationDecider() {
         }
     }
 
@@ -129,7 +128,7 @@ public void testRegisterAllocationDecider() {
             Collections.singletonList(new ClusterPlugin() {
                 @Override
                 public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
-                    return Collections.singletonList(new FakeAllocationDecider(settings));
+                    return Collections.singletonList(new FakeAllocationDecider());
                 }
             }), clusterInfoService);
         assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class)));
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 4e39e12b7ced7..67f5871506cc5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -89,9 +88,9 @@ public class ShardStateActionTests extends ESTestCase {
     private ClusterService clusterService;
 
     private static class TestShardStateAction extends ShardStateAction {
-        TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService,
+        TestShardStateAction(ClusterService clusterService, TransportService transportService,
                              AllocationService allocationService, RoutingService routingService) {
-            super(settings, clusterService, transportService, allocationService, routingService, THREAD_POOL);
+            super(clusterService, transportService, allocationService, routingService, THREAD_POOL);
         }
 
         private Runnable onBeforeWaitForNewMasterAndRetry;
@@ -130,7 +129,7 @@ public void setUp() throws Exception {
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
-        shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
+        shardStateAction = new TestShardStateAction(clusterService, transportService, null, null);
         shardStateAction.setOnBeforeWaitForNewMasterAndRetry(() -> {
         });
         shardStateAction.setOnAfterWaitForNewMasterAndRetry(() -> {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java
index 05bd9eeab8c3d..f14401980765a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AliasValidatorTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.test.ESTestCase;
 
@@ -27,7 +26,7 @@
 
 public class AliasValidatorTests extends ESTestCase {
     public void testValidatesAliasNames() {
-        AliasValidator validator = new AliasValidator(Settings.EMPTY);
+        AliasValidator validator = new AliasValidator();
         Exception e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone(".", null));
         assertEquals("Invalid alias name [.]: must not be '.' or '..'", e.getMessage());
         e = expectThrows(InvalidAliasNameException.class, () -> validator.validateAliasStandalone("..", null));
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java
index d6c8707c1d76e..53266ecd4b6dc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ClusterNameExpressionResolverTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -29,7 +28,7 @@
 
 public class ClusterNameExpressionResolverTests extends ESTestCase {
 
-    private ClusterNameExpressionResolver clusterNameResolver = new ClusterNameExpressionResolver(Settings.EMPTY);
+    private ClusterNameExpressionResolver clusterNameResolver = new ClusterNameExpressionResolver();
     private static final Set<String> remoteClusters = new HashSet<>();
 
     static {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index a3c2aeb337400..2ddafba33d000 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -35,12 +35,15 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.function.Function;
@@ -526,8 +529,12 @@ public void testIndexOptionsEmptyCluster() {
     }
 
     private static IndexMetaData.Builder indexBuilder(String index) {
-        return IndexMetaData.builder(index).settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        return IndexMetaData.builder(index).settings(settings());
+    }
+
+    private static Settings.Builder settings() {
+        return settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
     }
 
     public void testConcreteIndicesIgnoreIndicesOneMissingIndex() {
@@ -665,7 +672,7 @@ public void testConcreteIndicesWildcardAndAliases() {
 
         // when ignoreAliases option is set, concreteIndexNames resolves the provided expressions
         // only against the defined indices
-        IndicesOptions ignoreAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
+        IndicesOptions ignoreAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
 
         String[] indexNamesIndexWildcard = indexNameExpressionResolver.concreteIndexNames(state, ignoreAliasesOptions, "foo*");
 
@@ -689,7 +696,7 @@ public void testConcreteIndicesWildcardAndAliases() {
 
         // when ignoreAliases option is not set, concreteIndexNames resolves the provided
         // expressions against the defined indices and aliases
-        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false);
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false, false);
 
         List<String> indexNames = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(state, indicesAndAliasesOptions, "foo*"));
         assertEquals(2, indexNames.size());
@@ -1192,13 +1199,13 @@ public void testDeleteIndexIgnoresAliases() {
         }
         {
             DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("test-alias");
-            deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(true, true, true, true, false, false, true));
+            deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(true, true, true, true, false, false, true, false));
             String[] indices = indexNameExpressionResolver.concreteIndexNames(state, deleteIndexRequest);
             assertEquals(0, indices.length);
         }
         {
             DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("test-a*");
-            deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, true, false, false, true));
+            deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, true, false, false, true, false));
             IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(state, deleteIndexRequest));
             assertEquals(infe.getIndex().getName(), "test-a*");
@@ -1312,4 +1319,69 @@ public void testInvalidIndex() {
             () -> indexNameExpressionResolver.concreteIndexNames(context, "_foo"));
         assertEquals("Invalid index name [_foo], must not start with '_'.", iine.getMessage());
     }
+
+    public void testIgnoreThrottled() {
+        MetaData.Builder mdBuilder = MetaData.builder()
+            .put(indexBuilder("test-index").state(State.OPEN)
+                .settings(settings().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true))
+                .putAlias(AliasMetaData.builder("test-alias")))
+            .put(indexBuilder("index").state(State.OPEN)
+                .putAlias(AliasMetaData.builder("test-alias2")))
+            .put(indexBuilder("index-closed").state(State.CLOSE)
+                .settings(settings().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true))
+                .putAlias(AliasMetaData.builder("test-alias-closed")));
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "*");
+            assertEquals(1, indices.length);
+            assertEquals("index", indices[0].getName());
+        }
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED, "test-alias");
+            assertEquals(1, indices.length);
+            assertEquals("test-index", indices[0].getName());
+        }
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-alias");
+            assertEquals(0, indices.length);
+        }
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "test-*");
+            assertEquals(1, indices.length);
+            assertEquals("index", indices[0].getName());
+        }
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, "ind*", "test-index");
+            assertEquals(2, indices.length);
+            Arrays.sort(indices, Comparator.comparing(Index::getName));
+            assertEquals("index", indices[0].getName());
+            assertEquals("test-index", indices[1].getName());
+        }
+
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES,
+                    IndicesOptions.Option.IGNORE_THROTTLED),
+                    EnumSet.of(IndicesOptions.WildcardStates.OPEN)), "ind*", "test-index");
+            assertEquals(2, indices.length);
+            Arrays.sort(indices, Comparator.comparing(Index::getName));
+            assertEquals("index", indices[0].getName());
+            assertEquals("test-index", indices[1].getName());
+        }
+        {
+            Index[] indices = indexNameExpressionResolver.concreteIndices(state,
+                new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                    EnumSet.of(IndicesOptions.WildcardStates.OPEN, IndicesOptions.WildcardStates.CLOSED)), "ind*", "test-index");
+            assertEquals(3, indices.length);
+            Arrays.sort(indices, Comparator.comparing(Index::getName));
+            assertEquals("index", indices[0].getName());
+            assertEquals("index-closed", indices[1].getName());
+            assertEquals("test-index", indices[2].getName());
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 336075f23bbb2..c13529b4ede2f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -156,8 +156,8 @@ public void testValidateShrinkIndex() {
         ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0,
             Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
-        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        AllocationService service = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
@@ -226,8 +226,8 @@ public void testValidateSplitIndex() {
         ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0,
             Settings.builder().put("index.blocks.write", true).put("index.number_of_routing_shards", targetShards).build()))
             .nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
-        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        AllocationService service = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
@@ -360,9 +360,7 @@ private void runPrepareResizeIndexSettingsTest(
                         .build();
 
         final AllocationService service = new AllocationService(
-                Settings.builder().build(),
-                new AllocationDeciders(Settings.EMPTY,
-                Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+                new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
                 new TestGatewayAllocator(),
                 new BalancedShardsAllocator(Settings.EMPTY),
                 EmptyClusterInfoService.INSTANCE);
@@ -415,7 +413,8 @@ public void testValidateIndexName() throws Exception {
 
         MetaDataCreateIndexService.validateIndexName("foo:bar", ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING
                         .getDefault(Settings.EMPTY)).build());
-        assertWarnings("index or alias name [foo:bar] containing ':' is deprecated and will not be supported in Elasticsearch 7.0+");
+        assertWarnings("index or alias name [foo:bar] containing ':' is deprecated. Elasticsearch 7.x will read, " +
+                        "but not allow creation of new indices containing ':'");
     }
 
     private void validateIndexName(String indexName, String errorMessage) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
index e1fbc47c4a022..9b2d58ac28758 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
@@ -43,9 +43,9 @@
 import static org.mockito.Mockito.when;
 
 public class MetaDataIndexAliasesServiceTests extends ESTestCase {
-    private final AliasValidator aliasValidator = new AliasValidator(Settings.EMPTY);
+    private final AliasValidator aliasValidator = new AliasValidator();
     private final MetaDataDeleteIndexService deleteIndexService = mock(MetaDataDeleteIndexService.class);
-    private final MetaDataIndexAliasesService service = new MetaDataIndexAliasesService(Settings.EMPTY, null, null, aliasValidator,
+    private final MetaDataIndexAliasesService service = new MetaDataIndexAliasesService(null, null, aliasValidator,
             deleteIndexService, xContentRegistry());
 
     public MetaDataIndexAliasesServiceTests() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
index 776a0a158aeb4..234e6e4d37b7b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -106,7 +105,7 @@ public void testCalculateChangesAddChangeAndDelete() {
             IndexTemplateMetaData.builder("changed_test_template").patterns(randomIndexPatterns()).build()
         );
 
-        final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, null, clusterService, threadPool,
+        final TemplateUpgradeService service = new TemplateUpgradeService(null, clusterService, threadPool,
             Arrays.asList(
                 templates -> {
                     if (shouldAdd) {
@@ -206,7 +205,7 @@ public void testUpdateTemplates() {
             additions.put("add_template_" + i, new BytesArray("{\"index_patterns\" : \"*\", \"order\" : " + i + "}"));
         }
 
-        final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool,
+        final TemplateUpgradeService service = new TemplateUpgradeService(mockClient, clusterService, threadPool,
             Collections.emptyList());
 
         IllegalStateException ise = expectThrows(IllegalStateException.class, () -> service.upgradeTemplates(additions, deletions));
@@ -298,7 +297,7 @@ public void testClusterStateUpdate() throws InterruptedException {
             return null;
         }).when(mockIndicesAdminClient).deleteTemplate(any(DeleteIndexTemplateRequest.class), any(ActionListener.class));
 
-        final TemplateUpgradeService service = new TemplateUpgradeService(Settings.EMPTY, mockClient, clusterService, threadPool,
+        final TemplateUpgradeService service = new TemplateUpgradeService(mockClient, clusterService, threadPool,
             Arrays.asList(
                 templates -> {
                     assertNull(templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template")
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
index 19e14605389b6..734712eb5f279 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
@@ -153,16 +153,16 @@ public void testResolveAliases() {
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
         // when ignoreAliases option is not set, WildcardExpressionResolver resolves the provided
         // expressions against the defined indices and aliases
-        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
-            true, false, true, false, false);
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
+            false, false);
         IndexNameExpressionResolver.Context indicesAndAliasesContext =
             new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
         // ignoreAliases option is set, WildcardExpressionResolver throws error when
-        IndicesOptions skipAliasesIndicesOptions = IndicesOptions.fromOptions(true, true, true, false, true, false, true);
+        IndicesOptions skipAliasesIndicesOptions = IndicesOptions.fromOptions(true, true, true, false, true, false, true, false);
         IndexNameExpressionResolver.Context skipAliasesLenientContext =
             new IndexNameExpressionResolver.Context(state, skipAliasesIndicesOptions);
         // ignoreAliases option is set, WildcardExpressionResolver resolves the provided expressions only against the defined indices
-        IndicesOptions errorOnAliasIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
+        IndicesOptions errorOnAliasIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
         IndexNameExpressionResolver.Context skipAliasesStrictContext =
             new IndexNameExpressionResolver.Context(state, errorOnAliasIndicesOptions);
 
@@ -217,13 +217,13 @@ public void testMatchesConcreteIndicesWildcardAndAliases() {
 
         // when ignoreAliases option is not set, WildcardExpressionResolver resolves the provided
         // expressions against the defined indices and aliases
-        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false);
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false, false);
         IndexNameExpressionResolver.Context indicesAndAliasesContext =
             new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
 
         // ignoreAliases option is set, WildcardExpressionResolver resolves the provided expressions
         // only against the defined indices
-        IndicesOptions onlyIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true);
+        IndicesOptions onlyIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
         IndexNameExpressionResolver.Context onlyIndicesContext = new IndexNameExpressionResolver.Context(state, onlyIndicesOptions);
 
         {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index 804b7da236843..d6f5526ff7ddc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -539,7 +539,7 @@ public void testAdaptiveReplicaSelection() throws Exception {
         Set<String> selectedNodes = new HashSet<>(numShards);
         TestThreadPool threadPool = new TestThreadPool("testThatOnlyNodesSupportNodeIds");
         ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
-        ResponseCollectorService collector = new ResponseCollectorService(Settings.EMPTY, clusterService);
+        ResponseCollectorService collector = new ResponseCollectorService(clusterService);
         Map<String, Long> outstandingRequests = new HashMap<>();
         GroupShardsIterator<ShardIterator> groupIterator = opRouting.searchShards(state,
                 indexNames, null, null, collector, outstandingRequests);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index d579b65e98895..c7d04a61c3fd6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -574,7 +574,7 @@ public void testMoveShardToNonDataNode() {
 
         Index index = clusterState.getMetaData().index("test").getIndex();
         MoveAllocationCommand command = new MoveAllocationCommand(index.getName(), 0, "node1", "node2");
-        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Collections.emptyList()),
             new RoutingNodes(clusterState, false), clusterState, ClusterInfo.EMPTY, System.nanoTime());
         logger.info("--> executing move allocation command to non-data node");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> command.execute(routingAllocation, false));
@@ -612,7 +612,7 @@ public void testMoveShardFromNonDataNode() {
 
         Index index = clusterState.getMetaData().index("test").getIndex();
         MoveAllocationCommand command = new MoveAllocationCommand(index.getName(), 0, "node2", "node1");
-        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Collections.emptyList()),
             new RoutingNodes(clusterState, false), clusterState, ClusterInfo.EMPTY, System.nanoTime());
         logger.info("--> executing move allocation command from non-data node");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> command.execute(routingAllocation, false));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index acf0b260e5d7b..0ef64e15ce7a2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -264,7 +264,7 @@ public void testPersistedSettings() {
 
     public void testNoRebalanceOnPrimaryOverload() {
         Settings.Builder settings = Settings.builder();
-        AllocationService strategy = new AllocationService(settings.build(), randomAllocationDeciders(settings.build(),
+        AllocationService strategy = new AllocationService(randomAllocationDeciders(settings.build(),
                 new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), random()),
                 new TestGatewayAllocator(), new ShardsAllocator() {
             /*
@@ -403,11 +403,6 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
     }
 
     private class NoopGatewayAllocator extends GatewayAllocator {
-
-        NoopGatewayAllocator() {
-            super(Settings.EMPTY);
-        }
-
         @Override
         public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
             // noop
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
index 405f459e99a39..e4a7fa47025d6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
@@ -61,7 +61,7 @@ public void testRebalanceNonStartedShardNotAllowed() {
             randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING));
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState)).getMoveDecision();
+            new AllocationDeciders(Collections.emptyList()), clusterState)).getMoveDecision();
         assertSame(MoveDecision.NOT_TAKEN, rebalanceDecision);
     }
 
@@ -70,7 +70,7 @@ public void testRebalanceNotAllowedDuringPendingAsyncFetch() {
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED);
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         RoutingAllocation routingAllocation = newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState);
+            new AllocationDeciders(Collections.emptyList()), clusterState);
         routingAllocation.setHasPendingAsyncFetch();
         MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
         assertNotNull(rebalanceDecision.getClusterRebalanceDecision());
@@ -85,7 +85,7 @@ public void testRebalanceNotAllowedDuringPendingAsyncFetch() {
 
     public void testRebalancingNotAllowedDueToCanRebalance() {
         final Decision canRebalanceDecision = randomFrom(Decision.NO, Decision.THROTTLE);
-        AllocationDecider noRebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider noRebalanceDecider = new AllocationDecider() {
             @Override
             public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
                 return allocation.decision(canRebalanceDecision, "TEST", "foobar");
@@ -95,7 +95,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED);
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         RoutingAllocation routingAllocation =  newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, Collections.singleton(noRebalanceDecider)), clusterState);
+            new AllocationDeciders(Collections.singleton(noRebalanceDecider)), clusterState);
         MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
         assertEquals(canRebalanceDecision.type(), rebalanceDecision.getClusterRebalanceDecision().type());
         assertEquals(AllocationDecision.fromDecisionType(canRebalanceDecision.type()), rebalanceDecision.getAllocationDecision());
@@ -112,7 +112,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
     }
 
     public void testRebalancePossible() {
-        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider canAllocateDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                 return Decision.YES;
@@ -127,7 +127,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     }
 
     public void testRebalancingNotAllowedDueToCanAllocate() {
-        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider canAllocateDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                 return Decision.NO;
@@ -150,7 +150,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     }
 
     public void testDontBalanceShardWhenThresholdNotMet() {
-        AllocationDecider canAllocateDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider canAllocateDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                 return Decision.YES;
@@ -199,7 +199,7 @@ public void testSingleShardBalanceProducesSameResultsAsBalanceStep() {
         }
         clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
 
-        AllocationDecider allocationDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider allocationDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                 if (excludeNodes.contains(node.nodeId())) {
@@ -208,7 +208,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 return Decision.YES;
             }
         };
-        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider rebalanceDecider = new AllocationDecider() {
             @Override
             public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
                 return Decision.YES;
@@ -216,7 +216,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         };
         List<AllocationDecider> allocationDeciders = Arrays.asList(rebalanceDecider, allocationDecider);
         RoutingAllocation routingAllocation = newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+            new AllocationDeciders(allocationDeciders), clusterState);
         // allocate and get the node that is now relocating
         BalancedShardsAllocator allocator = new BalancedShardsAllocator(Settings.EMPTY);
         allocator.allocate(routingAllocation);
@@ -229,7 +229,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
             }
         }
 
-        routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+        routingAllocation = newRoutingAllocation(new AllocationDeciders(allocationDeciders), clusterState);
         routingAllocation.debugDecision(true);
         ShardRouting shard = clusterState.getRoutingNodes().activePrimary(shardToRebalance.shardId());
         MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
@@ -307,7 +307,7 @@ private MoveDecision executeRebalanceFor(final ShardRouting shardRouting, final
         if (Float.compare(-1.0f, threshold) != 0) {
             settings = Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), threshold).build();
         }
-        AllocationDecider allocationDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider allocationDecider = new AllocationDecider() {
             @Override
             public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                 if (noDecisionNodes.contains(node.nodeId())) {
@@ -316,7 +316,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 return Decision.YES;
             }
         };
-        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider rebalanceDecider = new AllocationDecider() {
             @Override
             public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
                 return Decision.YES;
@@ -324,7 +324,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         };
         BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings);
         RoutingAllocation routingAllocation = newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, Arrays.asList(allocationDecider, rebalanceDecider)), clusterState);
+            new AllocationDeciders(Arrays.asList(allocationDecider, rebalanceDecider)), clusterState);
         return allocator.decideShardAllocation(shardRouting, routingAllocation).getMoveDecision();
     }
 
@@ -340,7 +340,7 @@ private ClusterState addNodesToClusterState(ClusterState clusterState, int numNo
     private Tuple<ClusterState, MoveDecision> setupStateAndRebalance(AllocationDecider allocationDecider,
                                                                      Settings balancerSettings,
                                                                      boolean rebalanceExpected) {
-        AllocationDecider rebalanceDecider = new AllocationDecider(Settings.EMPTY) {
+        AllocationDecider rebalanceDecider = new AllocationDecider() {
             @Override
             public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
                 return Decision.YES;
@@ -356,7 +356,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         RoutingAllocation routingAllocation = newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
+            new AllocationDeciders(allocationDeciders), clusterState);
         MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
 
         if (rebalanceExpected == false) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
index bee2275743bac..8c643c4b1c27e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
@@ -159,8 +159,7 @@ private ClusterState runAllocationTest(final Settings settings,
     }
 
     private static AllocationService newAllocationService(Settings settings, Set<AllocationDecider> deciders) {
-        return new AllocationService(settings,
-                                     new AllocationDeciders(settings, deciders),
+        return new AllocationService(new AllocationDeciders(deciders),
                                      new TestGatewayAllocator(),
                                      new BalancedShardsAllocator(settings),
                                      EmptyClusterInfoService.INSTANCE);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
index 994ee8f1438a1..2ce0b7b89bec2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -54,8 +54,8 @@ public class MaxRetryAllocationDeciderTests extends ESAllocationTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+        strategy = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new MaxRetryAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
     }
 
@@ -175,7 +175,7 @@ public void testFailedAllocation() {
             assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), i+1);
             assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom" + i));
             // MaxRetryAllocationDecider#canForceAllocatePrimary should return YES decisions because canAllocate returns YES here
-            assertEquals(Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY).canForceAllocatePrimary(
+            assertEquals(Decision.YES, new MaxRetryAllocationDecider().canForceAllocatePrimary(
                 unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0)));
         }
         // now we go and check that we are actually stick to unassigned on the next failure
@@ -193,7 +193,7 @@ public void testFailedAllocation() {
             assertEquals(unassignedPrimary.state(), UNASSIGNED);
             assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom"));
             // MaxRetryAllocationDecider#canForceAllocatePrimary should return a NO decision because canAllocate returns NO here
-            assertEquals(Decision.NO, new MaxRetryAllocationDecider(Settings.EMPTY).canForceAllocatePrimary(
+            assertEquals(Decision.NO, new MaxRetryAllocationDecider().canForceAllocatePrimary(
                 unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0)));
         }
 
@@ -215,7 +215,7 @@ public void testFailedAllocation() {
         assertEquals(unassignedPrimary.state(), INITIALIZING);
         assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom"));
         // bumped up the max retry count, so canForceAllocatePrimary should return a YES decision
-        assertEquals(Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY).canForceAllocatePrimary(
+        assertEquals(Decision.YES, new MaxRetryAllocationDecider().canForceAllocatePrimary(
             routingTable.index("idx").shard(0).shards().get(0), null, new RoutingAllocation(null, null, clusterState, null, 0)));
 
         // now we start the shard
@@ -242,7 +242,7 @@ public void testFailedAllocation() {
         assertEquals(unassignedPrimary.state(), UNASSIGNED);
         assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("ZOOOMG"));
         // Counter reset, so MaxRetryAllocationDecider#canForceAllocatePrimary should return a YES decision
-        assertEquals(Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY).canForceAllocatePrimary(
+        assertEquals(Decision.YES, new MaxRetryAllocationDecider().canForceAllocatePrimary(
             unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0)));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 988e46b2804e5..6aa3495b840f8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -340,9 +340,9 @@ public void testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNode
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().add(newNode).add(oldNode1).add(oldNode2)).build();
-        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new NodeVersionAllocationDecider(Settings.EMPTY)));
-        AllocationService strategy = new MockAllocationService(Settings.EMPTY,
+        AllocationDeciders allocationDeciders = new AllocationDeciders(
+            Collections.singleton(new NodeVersionAllocationDecider()));
+        AllocationService strategy = new MockAllocationService(
             allocationDeciders,
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
@@ -373,10 +373,10 @@ public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
                 new SnapshotRecoverySource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
                 Version.CURRENT, "test")).build())
             .nodes(DiscoveryNodes.builder().add(newNode).add(oldNode1).add(oldNode2)).build();
-        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, Arrays.asList(
-            new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY),
-            new NodeVersionAllocationDecider(Settings.EMPTY)));
-        AllocationService strategy = new MockAllocationService(Settings.EMPTY,
+        AllocationDeciders allocationDeciders = new AllocationDeciders(Arrays.asList(
+            new ReplicaAfterPrimaryActiveAllocationDecider(),
+            new NodeVersionAllocationDecider()));
+        AllocationService strategy = new MockAllocationService(
             allocationDeciders,
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
@@ -470,7 +470,7 @@ public void testMessages() {
             null, 0);
         routingAllocation.debugDecision(true);
 
-        final NodeVersionAllocationDecider allocationDecider = new NodeVersionAllocationDecider(Settings.EMPTY);
+        final NodeVersionAllocationDecider allocationDecider = new NodeVersionAllocationDecider();
         Decision decision = allocationDecider.canAllocate(primaryShard, newNode, routingAllocation);
         assertThat(decision.type(), is(Decision.Type.YES));
         assertThat(decision.getExplanation(), is("the primary shard is new or already existed on the node"));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index 8a4cbf6318b37..5760859261888 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -58,10 +58,10 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
      * balance.*/
     public void testRandomDecisions() {
         RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(random());
-        AllocationService strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+        AllocationService strategy = new AllocationService(new AllocationDeciders(
                 new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY,
                         new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
-                    new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY), randomAllocationDecider))),
+                    new ReplicaAfterPrimaryActiveAllocationDecider(), randomAllocationDecider))),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         int indices = scaledRandomIntBetween(1, 20);
         Builder metaBuilder = MetaData.builder();
@@ -190,7 +190,6 @@ public static final class RandomAllocationDecider extends AllocationDecider {
         private final Random random;
 
         public RandomAllocationDecider(Random random) {
-            super(Settings.EMPTY);
             this.random = random;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
index b02319994b61f..52f9b5a6e0ac9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
@@ -57,8 +57,8 @@ public class ResizeAllocationDeciderTests extends ESAllocationTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
-            Collections.singleton(new ResizeAllocationDecider(Settings.EMPTY))),
+        strategy = new AllocationService(new AllocationDeciders(
+            Collections.singleton(new ResizeAllocationDecider())),
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
     }
 
@@ -110,7 +110,7 @@ private ClusterState createInitialClusterState(boolean startShards, Version node
 
     public void testNonResizeRouting() {
         ClusterState clusterState = createInitialClusterState(true);
-        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY);
+        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0);
         ShardRouting shardRouting = TestShardRouting.newShardRouting("non-resize", 0, null, true, ShardRoutingState.UNASSIGNED);
         assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation));
@@ -134,7 +134,7 @@ public void testShrink() { // we don't handle shrink yet
             .metaData(metaData).build();
         Index idx = clusterState.metaData().index("target").getIndex();
 
-        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY);
+        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0);
         ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, 0), null, true, ShardRoutingState.UNASSIGNED,
             RecoverySource.LocalShardsRecoverySource.INSTANCE);
@@ -162,7 +162,7 @@ public void testSourceNotActive() {
         Index idx = clusterState.metaData().index("target").getIndex();
 
 
-        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY);
+        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0);
         int shardId = randomIntBetween(0, 3);
         int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id();
@@ -202,7 +202,7 @@ public void testSourcePrimaryActive() {
         Index idx = clusterState.metaData().index("target").getIndex();
 
 
-        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY);
+        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0);
         int shardId = randomIntBetween(0, 3);
         int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id();
@@ -263,7 +263,7 @@ public void testAllocateOnOldNode() {
         Index idx = clusterState.metaData().index("target").getIndex();
 
 
-        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY);
+        ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0);
         int shardId = randomIntBetween(0, 3);
         int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
index f059125f1ea3d..9856bd064ca72 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
@@ -105,7 +105,7 @@ public void testForceAllocatePrimaryOnSameNodeNotAllowed() {
         Index index = clusterState.getMetaData().index("idx").getIndex();
         ShardRouting primaryShard = clusterState.routingTable().index(index).shard(0).primaryShard();
         RoutingNode routingNode = clusterState.getRoutingNodes().node(primaryShard.currentNodeId());
-        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+        RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(Collections.emptyList()),
             new RoutingNodes(clusterState, false), clusterState, ClusterInfo.EMPTY, System.nanoTime());
 
         // can't force allocate same shard copy to the same node
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 58d19fb61cf05..4e1a567d98540 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -94,7 +94,7 @@ public void testDiskThreshold() {
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
@@ -106,11 +106,8 @@ public ClusterInfo getClusterInfo() {
                 return clusterInfo;
             }
         };
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders,
+                new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -183,16 +180,13 @@ public ClusterInfo getClusterInfo() {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7)
                 .build();
 
-        deciders = new AllocationDeciders(Settings.EMPTY,
+        deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
-        strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
@@ -214,16 +208,13 @@ public ClusterInfo getClusterInfo() {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6)
                 .build();
 
-        deciders = new AllocationDeciders(Settings.EMPTY,
+        deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
-        strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         clusterState = strategy.reroute(clusterState, "reroute");
 
@@ -281,7 +272,7 @@ public void testDiskThresholdWithAbsoluteSizes() {
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
@@ -294,11 +285,8 @@ public ClusterInfo getClusterInfo() {
             }
         };
 
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
@@ -346,11 +334,8 @@ public ClusterInfo getClusterInfo() {
                 return clusterInfo2;
             }
         };
-        strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
@@ -406,16 +391,13 @@ public ClusterInfo getClusterInfo() {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "20b")
                 .build();
 
-        deciders = new AllocationDeciders(Settings.EMPTY,
+        deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
-        strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         clusterState = strategy.reroute(clusterState, "reroute");
         logShardStates(clusterState);
@@ -438,16 +420,13 @@ public ClusterInfo getClusterInfo() {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "30b")
                 .build();
 
-        deciders = new AllocationDeciders(Settings.EMPTY,
+        deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
                         makeDecider(diskSettings))));
 
-        strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         clusterState = strategy.reroute(clusterState, "reroute");
 
@@ -529,7 +508,7 @@ public void testDiskThresholdWithShardSizes() {
         ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(
                             Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
@@ -544,11 +523,8 @@ public ClusterInfo getClusterInfo() {
             }
         };
 
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -596,7 +572,7 @@ public void testUnknownDiskUsage() {
         ImmutableOpenMap<String, Long> shardSizes = shardSizesBuilder.build();
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(
                         new SameShardAllocationDecider(
                             Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
@@ -611,11 +587,8 @@ public ClusterInfo getClusterInfo() {
             }
         };
 
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
@@ -699,7 +672,7 @@ public void testShardRelocationsTakenIntoAccount() {
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
         DiskThresholdDecider decider = makeDecider(diskSettings);
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders deciders = new AllocationDeciders(
                 new HashSet<>(Arrays.asList(new SameShardAllocationDecider(
                     Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
                 ), decider)));
@@ -712,11 +685,8 @@ public ClusterInfo getClusterInfo() {
             }
         };
 
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
@@ -893,17 +863,14 @@ public ClusterInfo getClusterInfo() {
                 return clusterInfo;
             }
         };
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList(
+        AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(
                 new SameShardAllocationDecider(
                     Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
                 ),
                 diskThresholdDecider
         )));
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
         // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away
         // and therefor we will have sufficient disk space on node1.
         ClusterState result = strategy.reroute(clusterState, "reroute");
@@ -990,19 +957,15 @@ public ClusterInfo getClusterInfo() {
             }
         };
 
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList(
+        AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(
                 new SameShardAllocationDecider(
                     Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
                 ),
                 diskThresholdDecider
         )));
 
-        AllocationService strategy = new AllocationService(Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
-
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build(), deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), cis);
+        AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(),
+                new BalancedShardsAllocator(Settings.EMPTY), cis);
         ClusterState result = strategy.reroute(clusterState, "reroute");
 
         assertThat(result.routingTable().index("test").getShards().get(0).primaryShard().state(), equalTo(STARTED));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index d3c09e3069bfc..9e68a7c1927fe 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -104,7 +104,7 @@ public void testCanAllocateUsesMaxAvailableSpace() {
         shardSizes.put("[test][0][p]", 10L); // 10 bytes
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(),
             mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)),
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
             clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
         allocation.debugDecision(true);
         Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
@@ -159,7 +159,7 @@ public void testCannotAllocateDueToLackOfDiskResources() {
         shardSizes.put("[test][0][p]", shardSize);
         ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(),
             shardSizes.build(), ImmutableOpenMap.of());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)),
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
             clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
         allocation.debugDecision(true);
         Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
@@ -240,7 +240,7 @@ public void testCanRemainUsesLeastAvailableSpace() {
 
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(),
             shardSizes.build(), shardRoutingMap.build());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)),
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
             clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
         allocation.debugDecision(true);
         Decision decision = decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
index d91493c2a2578..31f56504fea23 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
@@ -53,11 +53,11 @@ public class FilterAllocationDeciderTests extends ESAllocationTestCase {
     public void testFilterInitialRecovery() {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY, clusterSettings);
-        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY,
+        AllocationDeciders allocationDeciders = new AllocationDeciders(
             Arrays.asList(filterAllocationDecider,
                 new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
-                new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY)));
-        AllocationService service = new AllocationService(Settings.builder().build(), allocationDeciders,
+                new ReplicaAfterPrimaryActiveAllocationDecider()));
+        AllocationService service = new AllocationService(allocationDeciders,
             new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
         ClusterState state = createInitialClusterState(service, Settings.builder().put("index.routing.allocation.initial_recovery._id",
             "node2").build());
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
index 49d69272af629..86190b107e5e7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -186,8 +185,8 @@ private ClusterState createInitialClusterState() {
     }
 
     private Decision executeAllocation(final ClusterState clusterState, final ShardRouting shardRouting) {
-        final AllocationDecider decider = new RestoreInProgressAllocationDecider(Settings.EMPTY);
-        final RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.singleton(decider)),
+        final AllocationDecider decider = new RestoreInProgressAllocationDecider();
+        final RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
             clusterState.getRoutingNodes(), clusterState, null, 0L);
         allocation.debugDecision(true);
 
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
index 17939f077665d..9c94bfab7acd6 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
@@ -41,7 +41,7 @@
 public class SettingsFilterTests extends ESTestCase {
     public void testAddingAndRemovingFilters() {
         HashSet<String> hashSet = new HashSet<>(Arrays.asList("foo", "bar", "baz"));
-        SettingsFilter settingsFilter = new SettingsFilter(Settings.EMPTY, hashSet);
+        SettingsFilter settingsFilter = new SettingsFilter(hashSet);
         assertEquals(settingsFilter.getPatterns(), hashSet);
     }
 
@@ -147,7 +147,7 @@ private void assertExpectedLogMessages(Consumer<Logger> consumer,
     }
 
     private void testFiltering(Settings source, Settings filtered, String... patterns) throws IOException {
-        SettingsFilter settingsFilter = new SettingsFilter(Settings.EMPTY, Arrays.asList(patterns));
+        SettingsFilter settingsFilter = new SettingsFilter(Arrays.asList(patterns));
 
         // Test using direct filtering
         Settings filteredSettings = settingsFilter.filter(source);
diff --git a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
index ec94018223693..6a263b7db4215 100644
--- a/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
+++ b/server/src/test/java/org/elasticsearch/common/time/DateFormattersTests.java
@@ -175,8 +175,9 @@ public void testEqualsAndHashcode() {
         assertThat(DateFormatters.forPattern("YYYY").withZone(ZoneId.of("CET")), not(equalTo(DateFormatters.forPattern("YYYY"))));
 
         // different locale, thus not equals
-        assertThat(DateFormatters.forPattern("YYYY").withLocale(randomLocale(random())),
-            not(equalTo(DateFormatters.forPattern("YYYY"))));
+        DateFormatter f1 = DateFormatters.forPattern("YYYY").withLocale(Locale.CANADA);
+        DateFormatter f2 = f1.withLocale(Locale.FRENCH);
+        assertThat(f1, not(equalTo(f2)));
 
         // different pattern, thus not equals
         assertThat(DateFormatters.forPattern("YYYY"), not(equalTo(DateFormatters.forPattern("YY"))));
diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java
index 8922a38ea1e78..8cf00c9cc5d30 100644
--- a/server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/zen/FileBasedUnicastHostsProviderTests.java
@@ -128,7 +128,7 @@ public void testEmptyUnicastHostsFileLegacyLocation() throws Exception {
 
     public void testUnicastHostsDoesNotExist() {
         final Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        final FileBasedUnicastHostsProvider provider = new FileBasedUnicastHostsProvider(settings, createTempDir().toAbsolutePath());
+        final FileBasedUnicastHostsProvider provider = new FileBasedUnicastHostsProvider(createTempDir().toAbsolutePath());
         final List<TransportAddress> addresses = provider.buildDynamicHosts((hosts, limitPortCounts) ->
             UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
                 TimeValue.timeValueSeconds(10)));
@@ -180,7 +180,7 @@ private List<TransportAddress> setupAndRunHostProvider(final List<String> hostEn
             writer.write(String.join("\n", hostEntries));
         }
 
-        return new FileBasedUnicastHostsProvider(settings, configPath).buildDynamicHosts((hosts, limitPortCounts) ->
+        return new FileBasedUnicastHostsProvider(configPath).buildDynamicHosts((hosts, limitPortCounts) ->
             UnicastZenPing.resolveHostsLists(executorService, logger, hosts, limitPortCounts, transportService,
                 TimeValue.timeValueSeconds(10)));
     }
diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 9e57382bb4bc8..a3ae6b07b19c9 100644
--- a/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -142,7 +142,7 @@ private void setupMasterServiceAndNodeJoinController(ClusterState initialState)
         }
         masterService = ClusterServiceUtils.createMasterService(threadPool, initialState);
         nodeJoinController = new NodeJoinController(masterService, createAllocationService(Settings.EMPTY),
-            new ElectMasterService(Settings.EMPTY), Settings.EMPTY);
+            new ElectMasterService(Settings.EMPTY));
     }
 
     public void testSimpleJoinAccumulation() throws InterruptedException, ExecutionException {
diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
index ac1719269e7ae..23b118ebbed7e 100644
--- a/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
@@ -267,7 +267,6 @@ private static MockPublishAction buildPublishClusterStateAction(
                 new DiscoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         return new MockPublishAction(
-                settings,
                 transportService,
                 namedWriteableRegistry,
                 listener,
@@ -873,9 +872,9 @@ public static class MockPublishAction extends PublishClusterStateAction {
         AtomicBoolean timeoutOnCommit = new AtomicBoolean();
         AtomicBoolean errorOnCommit = new AtomicBoolean();
 
-        public MockPublishAction(Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry,
+        public MockPublishAction(TransportService transportService, NamedWriteableRegistry namedWriteableRegistry,
                                  IncomingClusterStateListener listener, DiscoverySettings discoverySettings) {
-            super(settings, transportService, namedWriteableRegistry, listener, discoverySettings);
+            super(transportService, namedWriteableRegistry, listener, discoverySettings);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/server/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
index 4502e32e6d375..9593b58eae97c 100644
--- a/server/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
@@ -47,7 +47,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testCleanupWhenEmpty() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
@@ -58,7 +58,7 @@ public void testCleanupWhenEmpty() throws Exception {
     }
     public void testDanglingIndicesDiscovery() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
@@ -76,7 +76,7 @@ public void testDanglingIndicesDiscovery() throws Exception {
 
     public void testInvalidIndexFolder() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
@@ -100,7 +100,7 @@ public void testInvalidIndexFolder() throws Exception {
 
     public void testDanglingProcessing() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
@@ -144,7 +144,7 @@ public void testDanglingProcessing() throws Exception {
 
     public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
@@ -159,7 +159,6 @@ public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exceptio
     }
 
     private DanglingIndicesState createDanglingIndicesState(NodeEnvironment env, MetaStateService metaStateService) {
-        return new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null,
-            mock(ClusterService.class));
+        return new DanglingIndicesState(env, metaStateService, null, mock(ClusterService.class));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
index d0bf02e3c4e1f..938c28fe855c9 100644
--- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
@@ -38,7 +38,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteLoadIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             metaStateService.writeIndex("test_write", index);
@@ -48,14 +48,14 @@ public void testWriteLoadIndex() throws Exception {
 
     public void testLoadMissingIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
             assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
         }
     }
 
     public void testWriteLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -67,7 +67,7 @@ public void testWriteLoadGlobal() throws Exception {
 
     public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -83,7 +83,7 @@ public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
 
     public void testLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
+            MetaStateService metaStateService = new MetaStateService(env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             MetaData metaData = MetaData.builder()
diff --git a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index ac35d8d136e27..e6fc0c535dfc4 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -214,7 +214,7 @@ public void testFoundAllocationAndAllocating() {
      */
     public void testForceAllocatePrimary() {
         testAllocator.addData(node1, "allocId1", randomBoolean());
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, Arrays.asList(
+        AllocationDeciders deciders = new AllocationDeciders(Arrays.asList(
             // since the deciders return a NO decision for allocating a shard (due to the guaranteed NO decision from the second decider),
             // the allocator will see if it can force assign the primary, where the decision will be YES
             new TestAllocateDecision(randomBoolean() ? Decision.YES : Decision.NO), getNoDeciderThatAllowsForceAllocate()
@@ -235,7 +235,7 @@ public void testForceAllocatePrimary() {
     public void testDontAllocateOnNoOrThrottleForceAllocationDecision() {
         testAllocator.addData(node1, "allocId1", randomBoolean());
         boolean forceDecisionNo = randomBoolean();
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, Arrays.asList(
+        AllocationDeciders deciders = new AllocationDeciders(Arrays.asList(
             // since both deciders here return a NO decision for allocating a shard,
             // the allocator will see if it can force assign the primary, where the decision will be either NO or THROTTLE,
             // so the shard will remain un-initialized
@@ -258,7 +258,7 @@ public void testDontAllocateOnNoOrThrottleForceAllocationDecision() {
      */
     public void testDontForceAllocateOnThrottleDecision() {
         testAllocator.addData(node1, "allocId1", randomBoolean());
-        AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, Arrays.asList(
+        AllocationDeciders deciders = new AllocationDeciders(Arrays.asList(
             // since we have a NO decision for allocating a shard (because the second decider returns a NO decision),
             // the allocator will see if it can force assign the primary, and in this case,
             // the TestAllocateDecision's decision for force allocating is to THROTTLE (using
@@ -467,10 +467,6 @@ class TestAllocator extends PrimaryShardAllocator {
 
         private Map<DiscoveryNode, TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> data;
 
-        TestAllocator() {
-            super(Settings.EMPTY);
-        }
-
         public TestAllocator clear() {
             data = null;
             return this;
diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index a63a76e7154f1..d30f7eafce4a8 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -217,11 +217,11 @@ public void testNoOrThrottleDecidersRemainsInUnassigned() {
      * to wait till throttling on it is done.
      */
     public void testThrottleWhenAllocatingToMatchingNode() {
-        RoutingAllocation allocation = onePrimaryOnNode1And1Replica(new AllocationDeciders(Settings.EMPTY,
+        RoutingAllocation allocation = onePrimaryOnNode1And1Replica(new AllocationDeciders(
             Arrays.asList(new TestAllocateDecision(Decision.YES),
                 new SameShardAllocationDecider(
                     Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
-                new AllocationDecider(Settings.EMPTY) {
+                new AllocationDecider() {
                     @Override
                     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                         if (node.node().equals(node2)) {
@@ -356,10 +356,6 @@ class TestAllocator extends ReplicaShardAllocator {
         private Map<DiscoveryNode, TransportNodesListShardStoreMetaData.StoreFilesMetaData> data = null;
         private AtomicBoolean fetchDataCalled = new AtomicBoolean(false);
 
-        TestAllocator() {
-            super(Settings.EMPTY);
-        }
-
         public void clean() {
             data = null;
         }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index bc03565974ac6..33529b21962f1 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -21,7 +21,6 @@
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -5079,6 +5078,60 @@ public void testLastRefreshCheckpoint() throws Exception {
         assertThat(engine.lastRefreshedCheckpoint(), equalTo(engine.getLocalCheckpoint()));
     }
 
+    public void testLuceneSnapshotRefreshesOnlyOnce() throws Exception {
+        final MapperService mapperService = createMapperService("test");
+        final long maxSeqNo = randomLongBetween(10, 50);
+        final AtomicLong refreshCounter = new AtomicLong();
+        try (Store store = createStore();
+             InternalEngine engine = createEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(),
+                 null,
+                 new ReferenceManager.RefreshListener() {
+                     @Override
+                     public void beforeRefresh() throws IOException {
+                         refreshCounter.incrementAndGet();
+                     }
+
+                     @Override
+                     public void afterRefresh(boolean didRefresh) throws IOException {
+
+                     }
+                 }, null, () -> SequenceNumbers.NO_OPS_PERFORMED))) {
+            for (long seqNo = 0; seqNo <= maxSeqNo; seqNo++) {
+                final ParsedDocument doc = testParsedDocument("id_" + seqNo, null, testDocumentWithTextField("test"),
+                    new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                engine.index(replicaIndexForDoc(doc, 1, seqNo, randomBoolean()));
+            }
+
+            final long initialRefreshCount = refreshCounter.get();
+            final Thread[] snapshotThreads = new Thread[between(1, 3)];
+            CountDownLatch latch = new CountDownLatch(1);
+            for (int i = 0; i < snapshotThreads.length; i++) {
+                final long min = randomLongBetween(0, maxSeqNo - 5);
+                final long max = randomLongBetween(min, maxSeqNo);
+                snapshotThreads[i] = new Thread(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        throw new AssertionError(e);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        latch.await();
+                        Translog.Snapshot changes = engine.newChangesSnapshot("test", mapperService, min, max, true);
+                        changes.close();
+                    }
+                });
+                snapshotThreads[i].start();
+            }
+            latch.countDown();
+            for (Thread thread : snapshotThreads) {
+                thread.join();
+            }
+            assertThat(refreshCounter.get(), equalTo(initialRefreshCount + 1L));
+            assertThat(engine.lastRefreshedCheckpoint(), equalTo(maxSeqNo));
+        }
+    }
+
     public void testAcquireSearcherOnClosingEngine() throws Exception {
         engine.close();
         expectThrows(AlreadyClosedException.class, () -> engine.acquireSearcher("test"));
diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
index c194322f43ebf..467bf3447935d 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
@@ -294,4 +294,14 @@ private List<Translog.Operation> drainAll(Translog.Snapshot snapshot) throws IOE
         }
         return operations;
     }
+
+    public void testOverFlow() throws Exception {
+        long fromSeqNo = randomLongBetween(0, 5);
+        long toSeqNo = randomLongBetween(Long.MAX_VALUE - 5, Long.MAX_VALUE);
+        try (Translog.Snapshot snapshot = engine.newChangesSnapshot("test", mapperService, fromSeqNo, toSeqNo, true)) {
+            IllegalStateException error = expectThrows(IllegalStateException.class, () -> drainAll(snapshot));
+            assertThat(error.getMessage(),
+                containsString("Not all operations between from_seqno [" + fromSeqNo + "] and to_seqno [" + toSeqNo + "] found"));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
index d8df9eedb88c2..8a6ce2b09274f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java
@@ -69,7 +69,8 @@ public void testDefaultConfiguration() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
@@ -101,7 +102,8 @@ public void testCompletionAnalyzerSettings() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
@@ -135,7 +137,8 @@ public void testTypeParsing() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
@@ -159,11 +162,12 @@ public void testParsingMinimal() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("completion", "suggestion")
                         .endObject()),
@@ -179,11 +183,12 @@ public void testParsingFailure() throws Exception {
             .endObject().endObject()
             .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         MapperParsingException e = expectThrows(MapperParsingException.class, () ->
-            defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                             .startObject()
                             .field("completion", 1.0)
                             .endObject()),
@@ -212,10 +217,11 @@ public void testKeywordWithSubCompletionAndContext() throws Exception {
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .array("keywordfield", "key1", "key2", "key3")
                     .endObject()),
@@ -266,10 +272,11 @@ public void testCompletionWithContextAndSubCompletion() throws Exception {
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                         .startObject("suggest")
                             .array("input","timmy","starbucks")
@@ -321,10 +328,11 @@ public void testCompletionWithContextAndSubCompletionIndexByPath() throws Except
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .array("suggest", "timmy","starbucks")
                     .array("cat","cafe","food")
@@ -357,10 +365,11 @@ public void testKeywordWithSubCompletionAndStringInsert() throws Exception {
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("geofield", "drm3btev3e86")//"41.12,-71.34"
                     .endObject()),
@@ -387,10 +396,11 @@ public void testCompletionTypeWithSubCompletionFieldAndStringInsert() throws Exc
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("suggest", "suggestion")
                     .endObject()),
@@ -418,10 +428,11 @@ public void testCompletionTypeWithSubCompletionFieldAndObjectInsert() throws Exc
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("completion")
                     .array("input","New York", "NY")
@@ -455,10 +466,11 @@ public void testCompletionTypeWithSubKeywordFieldAndObjectInsert() throws Except
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .startObject("completion")
                         .array("input","New York", "NY")
@@ -494,10 +506,11 @@ public void testCompletionTypeWithSubKeywordFieldAndStringInsert() throws Except
             .endObject().endObject()
         );
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("completion", "suggestion")
                     .endObject()),
@@ -520,11 +533,12 @@ public void testParsingMultiValued() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .array("completion", "suggestion1", "suggestion2")
                         .endObject()),
@@ -543,11 +557,12 @@ public void testParsingWithWeight() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("completion")
                         .field("input", "suggestion")
@@ -568,11 +583,12 @@ public void testParsingMultiValueWithWeight() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("completion")
                         .array("input", "suggestion1", "suggestion2", "suggestion3")
@@ -635,11 +651,12 @@ public void testParsingFull() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("completion")
                         .startObject()
@@ -672,11 +689,12 @@ public void testParsingMixed() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
 
-        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument parsedDocument = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("completion")
                         .startObject()
@@ -712,10 +730,11 @@ public void testNonContextEnabledParsingWithContexts() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         try {
-            defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                             .startObject()
                             .startObject("field1")
                             .field("input", "suggestion1")
@@ -739,13 +758,14 @@ public void testFieldValueValidation() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         CharsRefBuilder charsRefBuilder = new CharsRefBuilder();
         charsRefBuilder.append("sugg");
         charsRefBuilder.setCharAt(2, '\u001F');
         try {
-            defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                             .startObject()
                             .field("completion", charsRefBuilder.get().toString())
                             .endObject()),
@@ -759,8 +779,8 @@ public void testFieldValueValidation() throws Exception {
 
         charsRefBuilder.setCharAt(2, '\u0000');
         try {
-            defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                             .startObject()
                             .field("completion", charsRefBuilder.get().toString())
                             .endObject()),
@@ -774,8 +794,8 @@ public void testFieldValueValidation() throws Exception {
 
         charsRefBuilder.setCharAt(2, '\u001E');
         try {
-            defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                             .startObject()
                             .field("completion", charsRefBuilder.get().toString())
                             .endObject()),
@@ -788,8 +808,8 @@ public void testFieldValueValidation() throws Exception {
         }
 
         // empty inputs are ignored
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .array("completion", "   ", "")
                     .endObject()),
@@ -801,8 +821,8 @@ public void testFieldValueValidation() throws Exception {
         assertThat(ignoredFields.stringValue(), equalTo("completion"));
 
         // null inputs are ignored
-        ParsedDocument nullDoc = defaultMapper.parse(SourceToParse.source("test", "type1", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument nullDoc = defaultMapper.parse(SourceToParse.source("test", "type1", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .nullField("completion")
                     .endObject()),
@@ -819,7 +839,8 @@ public void testPrefixQueryType() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType().prefixQuery(new BytesRef("co"));
@@ -833,7 +854,8 @@ public void testFuzzyQueryType() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType().fuzzyQuery("co",
@@ -850,7 +872,8 @@ public void testRegexQueryType() throws Exception {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
         CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
         Query prefixQuery = completionFieldMapper.fieldType()
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java
index 587ac2e0605cc..1386d2e29a0fd 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldTypeTests.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.index.mapper.CompletionFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMappings;
 import org.junit.Before;
@@ -52,7 +50,8 @@ public void modify(MappedFieldType ft) {
             @Override
             public void modify(MappedFieldType ft) {
                 CompletionFieldMapper.CompletionFieldType cft = (CompletionFieldMapper.CompletionFieldType)ft;
-                ContextMappings contextMappings = new ContextMappings(Arrays.asList(ContextBuilder.category("foo").build(), ContextBuilder.geo("geo").build()));
+                ContextMappings contextMappings = new ContextMappings(Arrays.asList(ContextBuilder.category("foo").build(),
+                    ContextBuilder.geo("geo").build()));
                 cft.setContextMappings(contextMappings);
             }
         });
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
index a59dceacfedc9..4ae19af147a88 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
@@ -145,7 +145,8 @@ public void testCopyToFieldsInnerObjectParsing() throws Exception {
 
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
                 .field("copy_test", "foo")
@@ -172,7 +173,8 @@ public void testCopyToDynamicInnerObjectParsing() throws Exception {
             .endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
                 .field("copy_test", "foo")
@@ -209,7 +211,8 @@ public void testCopyToDynamicInnerInnerObjectParsing() throws Exception {
             .endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
             .field("copy_test", "foo")
@@ -239,7 +242,8 @@ public void testCopyToStrictDynamicInnerObjectParsing() throws Exception {
                 .endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
             .field("copy_test", "foo")
@@ -273,7 +277,8 @@ public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception {
             .endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
             .field("copy_test", "foo")
@@ -283,7 +288,8 @@ public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception {
             docMapper.parse(SourceToParse.source("test", "type1", "1", json, XContentType.JSON)).rootDoc();
             fail();
         } catch (MapperParsingException ex) {
-          assertThat(ex.getMessage(), startsWith("mapping set to strict, dynamic introduction of [field] within [very.far] is not allowed"));
+          assertThat(ex.getMessage(),
+              startsWith("mapping set to strict, dynamic introduction of [field] within [very.far] is not allowed"));
         }
     }
 
@@ -307,12 +313,14 @@ public void testCopyToFieldMerge() throws Exception {
                 .endObject().endObject().endObject());
 
         MapperService mapperService = createIndex("test").mapperService();
-        DocumentMapper docMapperBefore = mapperService.merge("type1", new CompressedXContent(mappingBefore), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper docMapperBefore = mapperService.merge("type1", new CompressedXContent(mappingBefore),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         FieldMapper fieldMapperBefore = (FieldMapper) docMapperBefore.mappers().getMapper("copy_test");
 
         assertEquals(Arrays.asList("foo", "bar"), fieldMapperBefore.copyTo().copyToFields());
 
-        DocumentMapper docMapperAfter = mapperService.merge("type1", new CompressedXContent(mappingAfter), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper docMapperAfter = mapperService.merge("type1", new CompressedXContent(mappingAfter),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         FieldMapper fieldMapperAfter = (FieldMapper) docMapperAfter.mappers().getMapper("copy_test");
 
         assertEquals(Arrays.asList("baz", "bar"), fieldMapperAfter.copyTo().copyToFields());
@@ -385,7 +393,8 @@ public void testCopyToNestedField() throws Exception {
                         .endArray()
                     .endObject();
 
-        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(jsonDoc), XContentType.JSON));
+        ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(jsonDoc), XContentType.JSON));
         assertEquals(6, doc.docs().size());
 
         Document nested = doc.docs().get(0);
@@ -544,7 +553,8 @@ public void testCopyToDynamicNestedObjectParsing() throws Exception {
             .endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
 
         BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
             .field("copy_test", "foo")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index a913a0f2ceb80..d0f9d57a86347 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -77,7 +77,7 @@ public void testDefaults() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(8, pointField.fieldType().pointNumBytes());
         assertFalse(pointField.fieldType().stored());
         assertEquals(1457654400000L, pointField.numericValue().longValue());
@@ -128,7 +128,7 @@ public void testNoDocValues() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
     }
 
     public void testStore() throws Exception {
@@ -150,7 +150,7 @@ public void testStore() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(3, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
         IndexableField storedField = fields[2];
@@ -304,7 +304,7 @@ public void testNullValue() throws IOException {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(8, pointField.fieldType().pointNumBytes());
         assertFalse(pointField.fieldType().stored());
         assertEquals(1457654400000L, pointField.numericValue().longValue());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
index 5a0f1b76cad19..3b64a327062f6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperMergeTests.java
@@ -58,7 +58,8 @@ public void test1Merge() throws Exception {
         String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("person").startObject("properties")
                 .startObject("name").field("type", "text").endObject()
                 .startObject("age").field("type", "integer").endObject()
-                .startObject("obj1").startObject("properties").startObject("prop1").field("type", "integer").endObject().endObject().endObject()
+                .startObject("obj1").startObject("properties").startObject("prop1").field("type", "integer")
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
         DocumentMapper stage2 = parser.parse("person", new CompressedXContent(stage2Mapping));
 
@@ -77,7 +78,8 @@ public void testMergeObjectDynamic() throws Exception {
         DocumentMapper mapper = parser.parse("type1", new CompressedXContent(objectMapping));
         assertNull(mapper.root().dynamic());
 
-        String withDynamicMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1").field("dynamic", "false").endObject().endObject());
+        String withDynamicMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
+            .field("dynamic", "false").endObject().endObject());
         DocumentMapper withDynamicMapper = parser.parse("type1", new CompressedXContent(withDynamicMapping));
         assertThat(withDynamicMapper.root().dynamic(), equalTo(ObjectMapper.Dynamic.FALSE));
 
@@ -160,7 +162,8 @@ public void testChangeSearchAnalyzerToDefault() throws Exception {
 
     public void testConcurrentMergeTest() throws Throwable {
         final MapperService mapperService = createIndex("test").mapperService();
-        mapperService.merge("test", new CompressedXContent("{\"test\":{}}"), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge("test", new CompressedXContent("{\"test\":{}}"),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         final DocumentMapper documentMapper = mapperService.documentMapper("test");
 
         DocumentFieldMappers dfm = documentMapper.mappers();
@@ -190,7 +193,8 @@ public void run() {
                         Mapping update = doc.dynamicMappingsUpdate();
                         assert update != null;
                         lastIntroducedFieldName.set(fieldName);
-                        mapperService.merge("test", new CompressedXContent(update.toString()), MapperService.MergeReason.MAPPING_UPDATE, false);
+                        mapperService.merge("test", new CompressedXContent(update.toString()),
+                            MapperService.MergeReason.MAPPING_UPDATE, false);
                     }
                 } catch (Exception e) {
                     error.set(e);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 5d78dd25be65c..5a98d7f83478b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -126,19 +126,22 @@ public void testUnexpectedFieldMappingType() throws Exception {
                 .endObject().endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
         {
-            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("foo", true).endObject());
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("foo", true)
+                .endObject());
             MapperException exception = expectThrows(MapperException.class,
                     () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
             assertThat(exception.getMessage(), containsString("failed to parse field [foo] of type [long]"));
         }
         {
-            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("bar", "bar").endObject());
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("bar", "bar")
+                .endObject());
             MapperException exception = expectThrows(MapperException.class,
                     () -> mapper.parse(SourceToParse.source("test", "type", "2", bytes, XContentType.JSON)));
             assertThat(exception.getMessage(), containsString("failed to parse field [bar] of type [boolean]"));
         }
         {
-            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("geo", 123).endObject());
+            BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("geo", 123)
+                .endObject());
             MapperException exception = expectThrows(MapperException.class,
                     () -> mapper.parse(SourceToParse.source("test", "type", "2", bytes, XContentType.JSON)));
             assertThat(exception.getMessage(), containsString("failed to parse field [geo] of type [geo_shape]"));
@@ -396,7 +399,8 @@ public void testDynamicLongArray() throws Exception {
 
     public void testDynamicFalseLongArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -411,7 +415,8 @@ public void testDynamicFalseLongArray() throws Exception {
 
     public void testDynamicStrictLongArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -428,7 +433,8 @@ public void testDynamicStrictLongArray() throws Exception {
     public void testMappedGeoPointArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("foo").field("type", "geo_point").field("doc_values", false)
+                .startObject("properties").startObject("foo").field("type", "geo_point")
+            .field("doc_values", false)
                 .endObject().endObject().endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -477,7 +483,8 @@ public void testDynamicObjectWithTemplate() throws Exception {
 
     public void testDynamicFalseObject() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -491,7 +498,8 @@ public void testDynamicFalseObject() throws Exception {
 
     public void testDynamicStrictObject() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -506,7 +514,8 @@ public void testDynamicStrictObject() throws Exception {
 
     public void testDynamicFalseValue() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -520,7 +529,8 @@ public void testDynamicFalseValue() throws Exception {
 
     public void testDynamicStrictValue() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -535,7 +545,8 @@ public void testDynamicStrictValue() throws Exception {
 
     public void testDynamicFalseNull() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -549,7 +560,8 @@ public void testDynamicFalseNull() throws Exception {
 
     public void testDynamicStrictNull() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -674,7 +686,8 @@ public void testDynamicDottedFieldNameLongArrayWithExistingParentWrongType() thr
 
     public void testDynamicFalseDottedFieldNameLongArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -689,7 +702,8 @@ public void testDynamicFalseDottedFieldNameLongArray() throws Exception {
 
     public void testDynamicStrictDottedFieldNameLongArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -793,7 +807,8 @@ public void testDynamicDottedFieldNameLongWithExistingParentWrongType() throws E
 
     public void testDynamicFalseDottedFieldNameLong() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -806,7 +821,8 @@ public void testDynamicFalseDottedFieldNameLong() throws Exception {
 
     public void testDynamicStrictDottedFieldNameLong() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -875,12 +891,12 @@ public void testDynamicDottedFieldNameObjectWithParentTemplate() throws Exceptio
 
     public void testDynamicDottedFieldNameObjectWithExistingParent() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("foo")
-                .field("type", "object").endObject().endObject().endObject().endObject());
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("foo").field("type", "object").endObject().endObject().endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
-        BytesReference bytes = BytesReference
-                .bytes(XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz").field("a", 0).endObject().endObject());
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz")
+            .field("a", 0).endObject().endObject());
         ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON));
         assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
@@ -905,8 +921,8 @@ public void testDynamicDottedFieldNameObjectWithExistingParentWrongType() throws
             .endObject().endObject().endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
-        BytesReference bytes = BytesReference
-                .bytes(XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz").field("a", 0).endObject().endObject());
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz")
+            .field("a", 0).endObject().endObject());
         MapperParsingException exception = expectThrows(MapperParsingException.class,
                 () -> mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
 
@@ -916,7 +932,8 @@ public void testDynamicDottedFieldNameObjectWithExistingParentWrongType() throws
 
     public void testDynamicFalseDottedFieldNameObject() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "false")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -930,7 +947,8 @@ public void testDynamicFalseDottedFieldNameObject() throws Exception {
 
     public void testDynamicStrictDottedFieldNameObject() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+            .field("dynamic", "strict")
             .endObject().endObject());
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
 
@@ -953,7 +971,8 @@ public void testDocumentContainsMetadataField() throws Exception {
             mapper.parse(SourceToParse.source("test", "type", "1", bytes, XContentType.JSON)));
         assertTrue(e.getMessage(), e.getMessage().contains("cannot be added inside a document"));
 
-        BytesReference bytes2 = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("foo._ttl", 0).endObject());
+        BytesReference bytes2 = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .field("foo._ttl", 0).endObject());
         mapper.parse(SourceToParse.source("test", "type", "1", bytes2, XContentType.JSON)); // parses without error
     }
 
@@ -961,7 +980,8 @@ public void testSimpleMapper() throws Exception {
         IndexService indexService = createIndex("test");
         DocumentMapper docMapper = new DocumentMapper.Builder(
                 new RootObjectMapper.Builder("person")
-                        .add(new ObjectMapper.Builder("name").add(new TextFieldMapper.Builder("first").store(true).index(false))),
+                        .add(new ObjectMapper.Builder("name")
+                            .add(new TextFieldMapper.Builder("first").store(true).index(false))),
             indexService.mapperService()).build(indexService.mapperService());
 
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json"));
@@ -986,7 +1006,8 @@ public void testParseToJsonAndParse() throws Exception {
 
     public void testSimpleParser() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("person", new CompressedXContent(mapping));
 
         assertThat((String) docMapper.meta().get("param1"), equalTo("value1"));
 
@@ -998,7 +1019,8 @@ public void testSimpleParser() throws Exception {
 
     public void testSimpleParserNoTypeNoId() throws Exception {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("person", new CompressedXContent(mapping));
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1-notype-noid.json"));
         Document doc = docMapper.parse(SourceToParse.source("test", "person", "1", json, XContentType.JSON)).rootDoc();
         assertThat(doc.getBinaryValue(docMapper.idFieldMapper().name()), equalTo(Uid.encodeId("1")));
@@ -1021,7 +1043,8 @@ public void testNoDocumentSent() throws Exception {
         IndexService indexService = createIndex("test");
         DocumentMapper docMapper = new DocumentMapper.Builder(
                 new RootObjectMapper.Builder("person")
-                        .add(new ObjectMapper.Builder("name").add(new TextFieldMapper.Builder("first").store(true).index(false))),
+                        .add(new ObjectMapper.Builder("name")
+                            .add(new TextFieldMapper.Builder("first").store(true).index(false))),
             indexService.mapperService()).build(indexService.mapperService());
 
         BytesReference json = new BytesArray("".getBytes(StandardCharsets.UTF_8));
@@ -1036,7 +1059,8 @@ public void testNoDocumentSent() throws Exception {
     public void testNoLevel() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1055,7 +1079,8 @@ public void testNoLevel() throws Exception {
     public void testTypeLevel() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1074,7 +1099,8 @@ public void testTypeLevel() throws Exception {
     public void testNoLevelWithFieldTypeAsValue() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1095,7 +1121,8 @@ public void testNoLevelWithFieldTypeAsValue() throws Exception {
     public void testTypeLevelWithFieldTypeAsValue() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1116,7 +1143,8 @@ public void testTypeLevelWithFieldTypeAsValue() throws Exception {
     public void testNoLevelWithFieldTypeAsObject() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1137,7 +1165,8 @@ public void testNoLevelWithFieldTypeAsObject() throws Exception {
     public void testTypeLevelWithFieldTypeAsObject() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1158,7 +1187,8 @@ public void testTypeLevelWithFieldTypeAsObject() throws Exception {
     public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1179,7 +1209,8 @@ public void testNoLevelWithFieldTypeAsValueNotFirst() throws Exception {
     public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1200,7 +1231,8 @@ public void testTypeLevelWithFieldTypeAsValueNotFirst() throws Exception {
     public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1222,7 +1254,8 @@ public void testNoLevelWithFieldTypeAsObjectNotFirst() throws Exception {
     public void testTypeLevelWithFieldTypeAsObjectNotFirst() throws Exception {
         String defaultMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(defaultMapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -1313,7 +1346,7 @@ public void testDynamicFieldsStartingAndEndingWithDot() throws Exception {
             fail("should have failed to dynamically introduce a double-dot field");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(),
-                    containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]"));
+                containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]"));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 81a9734940b05..06a5c87d52f07 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -68,7 +68,8 @@ public void testDynamicTrue() throws IOException {
                 .endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(jsonBuilder()
@@ -90,7 +91,8 @@ public void testDynamicFalse() throws IOException {
                 .endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(jsonBuilder()
@@ -113,10 +115,12 @@ public void testDynamicStrict() throws IOException {
                 .endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () -> defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(jsonBuilder()
+        StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class,
+            () -> defaultMapper.parse(SourceToParse.source("test", "type", "1",
+                BytesReference.bytes(jsonBuilder()
                         .startObject()
                         .field("field1", "value1")
                         .field("field2", "value2")
@@ -124,8 +128,9 @@ public void testDynamicStrict() throws IOException {
                 XContentType.JSON)));
         assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed"));
 
-        e = expectThrows(StrictDynamicMappingException.class, () -> defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+        e = expectThrows(StrictDynamicMappingException.class,
+            () -> defaultMapper.parse(SourceToParse.source("test", "type", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                                 .startObject()
                                 .field("field1", "value1")
                                 .field("field2", (String) null)
@@ -144,10 +149,11 @@ public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOExcept
                 .endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(jsonBuilder()
                         .startObject().startObject("obj1")
                         .field("field1", "value1")
                         .field("field2", "value2")
@@ -169,11 +175,12 @@ public void testDynamicStrictWithInnerObjectButDynamicSetOnRoot() throws IOExcep
                 .endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () ->
-            defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                    .bytes(jsonBuilder()
+            defaultMapper.parse(SourceToParse.source("test", "type", "1",
+                BytesReference.bytes(jsonBuilder()
                             .startObject().startObject("obj1")
                             .field("field1", "value1")
                             .field("field2", "value2")
@@ -225,7 +232,8 @@ private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XConten
             .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
             .numberOfShards(1).numberOfReplicas(0).build();
         IndexSettings settings = new IndexSettings(build, Settings.EMPTY);
-        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", BytesReference.bytes(builder), builder.contentType());
+        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id",
+            BytesReference.bytes(builder), builder.contentType());
         try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) {
             ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser);
             assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
@@ -289,7 +297,8 @@ public void testIncremental() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
 
-        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject());
+        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar")
+            .field("bar", "baz").endObject());
         assertNotNull(update);
         // original mapping not modified
         assertEquals(mapping, serialize(mapper));
@@ -317,7 +326,8 @@ public void testIntroduceTwoFields() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
 
-        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").field("bar", "baz").endObject());
+        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar")
+            .field("bar", "baz").endObject());
         assertNotNull(update);
         // original mapping not modified
         assertEquals(mapping, serialize(mapper));
@@ -352,14 +362,17 @@ public void testObject() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
 
-        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject());
+        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar")
+            .field("baz", "foo").endObject().endObject().endObject());
         assertNotNull(update);
         // original mapping not modified
         assertEquals(mapping, serialize(mapper));
         // but we have an update
         assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text")
-                .startObject("fields").startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject()
+                .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz")
+                .field("type", "text")
+                .startObject("fields").startObject("keyword").field("type", "keyword")
+                .field("ignore_above", 256).endObject()
                 .endObject().endObject().endObject().endObject().endObject().endObject()
                 .endObject().endObject().endObject()), serialize(update));
     }
@@ -374,7 +387,8 @@ public void testArray() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
 
-        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startArray("foo").value("bar").value("baz").endArray().endObject());
+        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject()
+            .startArray("foo").value("bar").value("baz").endArray().endObject());
         assertNotNull(update);
         // original mapping not modified
         assertEquals(mapping, serialize(mapper));
@@ -402,13 +416,15 @@ public void testInnerDynamicMapping() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
 
-        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo").startObject("bar").field("baz", "foo").endObject().endObject().endObject());
+        Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().startObject("foo")
+            .startObject("bar").field("baz", "foo").endObject().endObject().endObject());
         assertNotNull(update);
         // original mapping not modified
         assertEquals(mapping, serialize(mapper));
         // but we have an update
         assertEquals(Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("foo").startObject("properties").startObject("bar").startObject("properties").startObject("baz").field("type", "text").startObject("fields")
+                .startObject("foo").startObject("properties").startObject("bar").startObject("properties")
+                .startObject("baz").field("type", "text").startObject("fields")
                 .startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject()
                 .endObject().endObject().endObject().endObject().endObject().endObject()
                 .endObject().endObject().endObject()), serialize(update));
@@ -605,7 +621,8 @@ private void doTestDefaultFloatingPointMappings(DocumentMapper mapper, XContentB
                 .field("baz", (double) 3.2f) // double that can be accurately represented as a float
                 .field("quux", "3.2") // float detected through numeric detection
                 .endObject());
-        ParsedDocument parsedDocument = mapper.parse(SourceToParse.source("index", "type", "id", source, builder.contentType()));
+        ParsedDocument parsedDocument = mapper.parse(SourceToParse.source("index", "type", "id",
+            source, builder.contentType()));
         Mapping update = parsedDocument.dynamicMappingsUpdate();
         assertNotNull(update);
         assertThat(((FieldMapper) update.root().getMapper("foo")).fieldType().typeName(), equalTo("float"));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
index 3ae45aab49aed..e42504e8e9094 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
@@ -92,8 +92,8 @@ public ExternalMapper build(BuilderContext context) {
 
             setupFieldType(context);
 
-            return new ExternalMapper(name, fieldType, generatedValue, mapperName, binMapper, boolMapper, pointMapper, shapeMapper, stringMapper,
-                    context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
+            return new ExternalMapper(name, fieldType, generatedValue, mapperName, binMapper, boolMapper, pointMapper,
+                shapeMapper, stringMapper, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
         }
     }
 
@@ -156,7 +156,8 @@ public Query existsQuery(QueryShardContext context) {
     public ExternalMapper(String simpleName, MappedFieldType fieldType,
                           String generatedValue, String mapperName,
                           BinaryFieldMapper binMapper, BooleanFieldMapper boolMapper, GeoPointFieldMapper pointMapper,
-                          GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+                          GeoShapeFieldMapper shapeMapper, FieldMapper stringMapper, Settings indexSettings,
+                          MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, new ExternalFieldType(), indexSettings, multiFields, copyTo);
         this.generatedValue = generatedValue;
         this.mapperName = mapperName;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
index 2ae6345d44e6e..e8c38bc1a5a93 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java
@@ -24,13 +24,6 @@
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.BooleanFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MetadataFieldMapper;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -99,7 +92,8 @@ public ExternalMetadataMapper build(BuilderContext context) {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
 
         @Override
-        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node,
+                                                       ParserContext parserContext) throws MapperParsingException {
             return new Builder();
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index 32a2f73536a00..5653b0cb9c533 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -67,7 +67,8 @@ public void testFieldType() throws Exception {
             .startObject("_field_names").endObject()
             .endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
         FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class);
         assertFalse(fieldNamesMapper.fieldType().hasDocValues());
         assertEquals(IndexOptions.DOCS, fieldNamesMapper.fieldType().indexOptions());
@@ -78,10 +79,11 @@ public void testFieldType() throws Exception {
 
     public void testInjectIntoDocDuringParsing() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                             .field("a", "100")
                             .startObject("b")
@@ -96,14 +98,16 @@ public void testInjectIntoDocDuringParsing() throws Exception {
     public void testExplicitEnabled() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_field_names").field("enabled", true).endObject()
-                .startObject("properties").startObject("field").field("type", "keyword").field("doc_values", false).endObject().endObject()
-            .endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+            .startObject("properties")
+            .startObject("field").field("type", "keyword").field("doc_values", false).endObject()
+            .endObject().endObject().endObject());
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
         FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class);
         assertTrue(fieldNamesMapper.fieldType().isEnabled());
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-            .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
                 .endObject()),
@@ -116,12 +120,13 @@ public void testDisabled() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_field_names").field("enabled", false).endObject()
             .endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
         FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class);
         assertFalse(fieldNamesMapper.fieldType().isEnabled());
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-            .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
                 .endObject()),
@@ -139,11 +144,14 @@ public void testMergingMappings() throws Exception {
             .endObject().endObject());
         MapperService mapperService = createIndex("test").mapperService();
 
-        DocumentMapper mapperEnabled = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
-        DocumentMapper mapperDisabled = mapperService.merge("type", new CompressedXContent(disabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper mapperEnabled = mapperService.merge("type", new CompressedXContent(enabledMapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper mapperDisabled = mapperService.merge("type", new CompressedXContent(disabledMapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         assertFalse(mapperDisabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled());
 
-        mapperEnabled = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        mapperEnabled = mapperService.merge("type", new CompressedXContent(enabledMapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         assertTrue(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
index eabf0a849fa39..b3474fb5efbcd 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java
@@ -62,10 +62,11 @@ public void testGeoHashValue() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("point", stringEncode(1.3, 1.2))
                         .endObject()),
@@ -78,10 +79,11 @@ public void testLatLonValuesStored() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("point").field("lat", 1.2).field("lon", 1.3).endObject()
                         .endObject()),
@@ -94,10 +96,11 @@ public void testArrayLatLonValues() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point").field("doc_values", false);
         String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point")
                         .startObject().field("lat", 1.2).field("lon", 1.3).endObject()
@@ -115,11 +118,11 @@ public void testLatLonInOneValue() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type",
-            new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("point", "1.2,1.3")
                         .endObject()),
@@ -133,11 +136,11 @@ public void testLatLonStringWithZValue() throws Exception {
             .startObject("properties").startObject("point").field("type", "geo_point")
             .field(IGNORE_Z_VALUE.getPreferredName(), true);
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type",
-            new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-            .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject()
                 .field("point", "1.2,1.3,10.0")
                 .endObject()),
@@ -151,11 +154,11 @@ public void testLatLonStringWithZValueException() throws Exception {
             .startObject("properties").startObject("point").field("type", "geo_point")
             .field(IGNORE_Z_VALUE.getPreferredName(), false);
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type",
-            new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        SourceToParse source = SourceToParse.source("test", "type", "1", BytesReference
-            .bytes(XContentFactory.jsonBuilder()
+        SourceToParse source = SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject()
                 .field("point", "1.2,1.3,10.0")
                 .endObject()),
@@ -169,11 +172,11 @@ public void testLatLonInOneValueStored() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type",
-            new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("point", "1.2,1.3")
                         .endObject()),
@@ -185,11 +188,11 @@ public void testLatLonInOneValueArray() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point").field("doc_values", false);
         String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type",
-            new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point")
                         .value("1.2,1.3")
@@ -207,10 +210,11 @@ public void testLonLatArray() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point").value(1.3).value(1.2).endArray()
                         .endObject()),
@@ -224,10 +228,11 @@ public void testLonLatArrayDynamic() throws Exception {
             .startArray("dynamic_templates").startObject().startObject("point").field("match", "point*")
             .startObject("mapping").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.endObject().endObject().endObject().endArray().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point").value(1.3).value(1.2).endArray()
                         .endObject()),
@@ -240,10 +245,11 @@ public void testLonLatArrayStored() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
         String mapping = Strings.toString(xContentBuilder.field("store", true).endObject().endObject().endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point").value(1.3).value(1.2).endArray()
                         .endObject()),
@@ -256,12 +262,14 @@ public void testLonLatArrayStored() throws Exception {
     public void testLonLatArrayArrayStored() throws Exception {
         XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("properties").startObject("point").field("type", "geo_point");
-        String mapping = Strings.toString(xContentBuilder.field("store", true).field("doc_values", false).endObject().endObject()
+        String mapping = Strings.toString(xContentBuilder.field("store", true)
+            .field("doc_values", false).endObject().endObject()
             .endObject().endObject());
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startArray("point")
                         .startArray().value(1.3).value(1.2).endArray()
@@ -311,7 +319,8 @@ public void testIgnoreZValue() throws IOException {
 
     public void testMultiField() throws Exception {
         int numDocs = randomIntBetween(10, 100);
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("pin").startObject("properties").startObject("location")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("pin")
+            .startObject("properties").startObject("location")
             .field("type", "geo_point")
             .startObject("fields")
             .startObject("geohash").field("type", "keyword").endObject()  // test geohash as keyword
@@ -326,13 +335,15 @@ public void testMultiField() throws Exception {
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
         for (int i=0; i<numDocs; ++i) {
             final GeoPoint pt = RandomGeoGenerator.randomPoint(random());
-            client().prepareIndex("test", "pin").setSource(jsonBuilder().startObject().startObject("location").field("lat", pt.lat())
+            client().prepareIndex("test", "pin").setSource(jsonBuilder().startObject()
+                .startObject("location").field("lat", pt.lat())
                 .field("lon", pt.lon()).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
         }
 
         // TODO these tests are bogus and need to be Fix
         // query by geohash subfield
-        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash")
+            .setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().getTotalHits());
 
         // query by latlon subfield
@@ -370,8 +381,8 @@ public void testNullValue() throws Exception {
         Object nullValue = ((GeoPointFieldMapper) fieldMapper).fieldType().nullValue();
         assertThat(nullValue, equalTo(new GeoPoint(1, 2)));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .nullField("location")
                     .endObject()),
@@ -380,8 +391,8 @@ public void testNullValue() throws Exception {
         assertThat(doc.rootDoc().getField("location"), notNullValue());
         BytesRef defaultValue = doc.rootDoc().getField("location").binaryValue();
 
-        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("location", "1, 2")
                     .endObject()),
@@ -389,8 +400,8 @@ public void testNullValue() throws Exception {
         // Shouldn't matter if we specify the value explicitly or use null value
         assertThat(defaultValue, equalTo(doc.rootDoc().getField("location").binaryValue()));
 
-        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("location", "3, 4")
                     .endObject()),
@@ -411,8 +422,8 @@ public void testInvalidGeohashIgnored() throws Exception {
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
             .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("location", "1234.333")
                     .endObject()),
@@ -434,8 +445,8 @@ public void testInvalidGeohashNotIgnored() throws Exception {
             .parse("type", new CompressedXContent(mapping));
 
         MapperParsingException ex = expectThrows(MapperParsingException.class,
-            () -> defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+            () -> defaultMapper.parse(SourceToParse.source("test", "type", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                     .startObject()
                     .field("location", "1234.333")
                     .endObject()),
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
index 45609c1b0f031..51de034de0445 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
@@ -58,7 +58,8 @@ public void testDefaultConfiguration() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -82,7 +83,8 @@ public void testOrientationParsing() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -99,7 +101,8 @@ public void testOrientationParsing() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        defaultMapper = createIndex("test2").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -120,7 +123,8 @@ public void testCoerceParsing() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -135,7 +139,8 @@ public void testCoerceParsing() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        defaultMapper = createIndex("test2").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -171,7 +176,8 @@ public void testIgnoreZValue() throws IOException {
             .endObject().endObject()
             .endObject().endObject());
 
-        defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        defaultMapper = createIndex("test2").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -190,7 +196,8 @@ public void testIgnoreMalformedParsing() throws IOException {
             .endObject().endObject()
             .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -205,7 +212,8 @@ public void testIgnoreMalformedParsing() throws IOException {
             .endObject().endObject()
             .endObject().endObject());
 
-        defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        defaultMapper = createIndex("test2").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -224,7 +232,8 @@ public void testGeohashConfiguration() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -247,7 +256,8 @@ public void testQuadtreeConfiguration() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -394,7 +404,8 @@ public void testPointsOnlyOption() throws IOException {
                 .endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
@@ -455,14 +466,19 @@ public void testLevelDefaults() throws IOException {
 
     public void testGeoShapeMapperMerge() throws Exception {
         String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("shape").field("type", "geo_shape").field("tree", "geohash").field("strategy", "recursive")
-                .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01).field("orientation", "ccw")
+                .startObject("shape").field("type", "geo_shape").field("tree", "geohash")
+                .field("strategy", "recursive")
+                .field("precision", "1m").field("tree_levels", 8).field("distance_error_pct", 0.01)
+                .field("orientation", "ccw")
                 .endObject().endObject().endObject().endObject());
         MapperService mapperService = createIndex("test").mapperService();
-        DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper docMapper = mapperService.merge("type", new CompressedXContent(stage1Mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
         String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("shape").field("type", "geo_shape").field("tree", "quadtree")
-                .field("strategy", "term").field("precision", "1km").field("tree_levels", 26).field("distance_error_pct", 26)
+                .startObject("properties").startObject("shape").field("type", "geo_shape")
+                .field("tree", "quadtree")
+                .field("strategy", "term").field("precision", "1km")
+                .field("tree_levels", 26).field("distance_error_pct", 26)
                 .field("orientation", "cw").endObject().endObject().endObject().endObject());
         try {
             mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
@@ -490,8 +506,10 @@ public void testGeoShapeMapperMerge() throws Exception {
         // correct mapping
         stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("shape").field("type", "geo_shape").field("precision", "1m")
-                .field("tree_levels", 8).field("distance_error_pct", 0.001).field("orientation", "cw").endObject().endObject().endObject().endObject());
-        docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+                .field("tree_levels", 8).field("distance_error_pct", 0.001)
+                .field("orientation", "cw").endObject().endObject().endObject().endObject());
+        docMapper = mapperService.merge("type", new CompressedXContent(stage2Mapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
 
         fieldMapper = docMapper.mappers().getMapper("shape");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
@@ -599,7 +617,8 @@ public void testPointsOnlyDefaultsWithTermStrategy() throws IOException {
             .endObject().endObject()
             .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type1", new CompressedXContent(mapping));
         Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
         assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
index 1926ab63222a0..03ab76f927d84 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
@@ -37,7 +37,8 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testIncludeInObjectNotAllowed() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder()
@@ -51,8 +52,10 @@ public void testIncludeInObjectNotAllowed() throws Exception {
     public void testDefaultsSingleType() throws IOException {
         Settings indexSettings = Settings.EMPTY;
         MapperService mapperService = createIndex("test", indexSettings).mapperService();
-        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE, false);
-        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON));
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"),
+            MergeReason.MAPPING_UPDATE, false);
+        ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id",
+            new BytesArray("{}"), XContentType.JSON));
         IndexableField[] fields = document.rootDoc().getFields(IdFieldMapper.NAME);
         assertEquals(1, fields.length);
         assertEquals(IndexOptions.DOCS, fields[0].fieldType().indexOptions());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
index 5e60e248927d7..e60b097aaca36 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
@@ -44,10 +44,11 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     public void testDefaultDisabledIndexMapper() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .endObject()),
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index df60333c30dc8..1ad3cb8baa839 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -78,7 +78,7 @@ public void testDefaults() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(16, pointField.fieldType().pointNumBytes());
         assertFalse(pointField.fieldType().stored());
         assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue());
@@ -129,7 +129,7 @@ public void testNoDocValues() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue());
     }
 
@@ -152,7 +152,7 @@ public void testStore() throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(3, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType());
         IndexableField storedField = fields[2];
@@ -240,7 +240,7 @@ public void testNullValue() throws IOException {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(16, pointField.fieldType().pointNumBytes());
         assertFalse(pointField.fieldType().stored());
         assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
index e16b04748a18b..2c70b25d6a446 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java
@@ -71,7 +71,7 @@ public void testStoreCidr() throws Exception {
             IndexableField dvField = fields[0];
             assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
             IndexableField pointField = fields[1];
-            assertEquals(2, pointField.fieldType().pointDimensionCount());
+            assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
             IndexableField storedField = fields[2];
             assertTrue(storedField.fieldType().stored());
             String strVal =
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index 5594cae2d1ef8..76198eaaacf37 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.indices.InvalidTypeNameException;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -76,7 +75,8 @@ public void testTypeNameTooLong() {
                     .addMapping(type, field, "type=text")
                     .execute().actionGet();
         });
-        assertTrue(e.getMessage(), e.getMessage().contains("mapping type name [" + type + "] is too long; limit is length 255 but was [256]"));
+        assertTrue(e.getMessage(), e.getMessage().contains("mapping type name [" + type
+            + "] is too long; limit is length 255 but was [256]"));
     }
 
     public void testTypeValidation() {
@@ -91,9 +91,9 @@ public void testTypeValidation() {
 
     public void testIndexIntoDefaultMapping() throws Throwable {
         // 1. test implicit index creation
-        ExecutionException e = expectThrows(ExecutionException.class, () -> {
-            client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}", XContentType.JSON).execute().get();
-        });
+        ExecutionException e = expectThrows(ExecutionException.class,
+            () -> client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1")
+                .setSource("{}", XContentType.JSON).execute().get());
         Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause());
         if (throwable instanceof IllegalArgumentException) {
             assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage());
@@ -122,14 +122,15 @@ public void testIndexIntoDefaultMapping() throws Throwable {
      */
     public void testTotalFieldsLimit() throws Throwable {
         int totalFieldsLimit = randomIntBetween(1, 10);
-        Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), totalFieldsLimit).build();
+        Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), totalFieldsLimit)
+            .build();
         createIndex("test1", settings).mapperService().merge("type", createMappingSpecifyingNumberOfFields(totalFieldsLimit),
                 MergeReason.MAPPING_UPDATE, false);
 
         // adding one more field should trigger exception
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            createIndex("test2", settings).mapperService().merge("type", createMappingSpecifyingNumberOfFields(totalFieldsLimit + 1),
-                    MergeReason.MAPPING_UPDATE, false);
+            createIndex("test2", settings).mapperService().merge("type",
+                createMappingSpecifyingNumberOfFields(totalFieldsLimit + 1), MergeReason.MAPPING_UPDATE, false);
         });
         assertTrue(e.getMessage(),
                 e.getMessage().contains("Limit of total fields [" + totalFieldsLimit + "] in index [test2] has been exceeded"));
@@ -151,7 +152,8 @@ public void testMappingDepthExceedsLimit() throws Throwable {
         IndexService indexService1 = createIndex("test1",
                 Settings.builder().put(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING.getKey(), 1).build());
         // no exception
-        indexService1.mapperService().merge("type", createMappingSpecifyingNumberOfFields(1), MergeReason.MAPPING_UPDATE, false);
+        indexService1.mapperService().merge("type", createMappingSpecifyingNumberOfFields(1),
+            MergeReason.MAPPING_UPDATE, false);
 
         CompressedXContent objectMapping = new CompressedXContent(BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
                 .startObject("properties")
@@ -344,7 +346,8 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
             createIndex("test2",
                     Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfNonAliasFields).build())
-                            .mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, true);
+                            .mapperService().merge("type", new CompressedXContent(mapping),
+                MergeReason.MAPPING_UPDATE, true);
         });
         assertEquals("Limit of total fields [" + numberOfNonAliasFields + "] in index [test2] has been exceeded", e.getMessage());
     }
@@ -357,7 +360,7 @@ public void testForbidMultipleTypes() throws IOException {
         String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type2").endObject().endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> mapperService.merge("type2", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, randomBoolean()));
-        assertThat(e.getMessage(), Matchers.startsWith("Rejecting mapping update to [test] as the final mapping would have more than 1 type: "));
+        assertThat(e.getMessage(), startsWith("Rejecting mapping update to [test] as the final mapping would have more than 1 type: "));
     }
 
     /**
@@ -366,15 +369,17 @@ public void testForbidMultipleTypes() throws IOException {
      */
     public void testForbidMultipleTypesWithConflictingMappings() throws IOException {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("properties").startObject("field1").field("type", "integer_range").endObject().endObject().endObject().endObject());
+            .startObject("properties").startObject("field1").field("type", "integer_range")
+            .endObject().endObject().endObject().endObject());
         MapperService mapperService = createIndex("test").mapperService();
         mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
 
         String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type2")
-            .startObject("properties").startObject("field1").field("type", "integer").endObject().endObject().endObject().endObject());
+            .startObject("properties").startObject("field1").field("type", "integer")
+            .endObject().endObject().endObject().endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> mapperService.merge("type2", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, false));
-        assertThat(e.getMessage(), Matchers.startsWith("Rejecting mapping update to [test] as the final mapping would have more than 1 type: "));
+        assertThat(e.getMessage(), startsWith("Rejecting mapping update to [test] as the final mapping would have more than 1 type: "));
     }
 
     public void testDefaultMappingIsDeprecated() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
index b17abcc17b359..6ecf4b6408be3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
@@ -43,7 +43,8 @@ public void testExceptionForCopyToInMultiFields() throws IOException {
             mapperService.parse("type", new CompressedXContent(Strings.toString(mapping)), true);
             fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field");
         } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field."));
+            assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c]"
+                + " which is within a multi field."));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
index da9e84cfcf4d7..ce04ff6489059 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
@@ -133,7 +133,8 @@ public void testBuildThenParse() throws Exception {
 
         String builtMapping = builderDocMapper.mappingSource().string();
         // reparse it
-        DocumentMapper docMapper = indexService.mapperService().documentMapperParser().parse("person", new CompressedXContent(builtMapping));
+        DocumentMapper docMapper = indexService.mapperService().documentMapperParser()
+            .parse("person", new CompressedXContent(builtMapping));
 
 
         BytesReference json = new BytesArray(copyToBytesFromClasspath("/org/elasticsearch/index/mapper/multifield/test-data.json"));
@@ -174,13 +175,15 @@ public void testMultiFieldsInConsistentOrder() throws Exception {
         }
         builder = builder.endObject().endObject().endObject().endObject().endObject();
         String mapping = Strings.toString(builder);
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
         Arrays.sort(multiFieldNames);
 
         Map<String, Object> sourceAsMap =
             XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true, builder.contentType()).v2();
         @SuppressWarnings("unchecked")
-        Map<String, Object> multiFields = (Map<String, Object>) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap);
+        Map<String, Object> multiFields =
+            (Map<String, Object>) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap);
         assertThat(multiFields.size(), equalTo(multiFieldNames.length));
 
         int i = 0;
@@ -192,7 +195,8 @@ public void testMultiFieldsInConsistentOrder() throws Exception {
 
     public void testObjectFieldNotAllowed() throws Exception {
         String mapping = Strings.toString(jsonBuilder().startObject().startObject("type").startObject("properties").startObject("my_field")
-            .field("type", "text").startObject("fields").startObject("multi").field("type", "object").endObject().endObject()
+            .field("type", "text").startObject("fields").startObject("multi").field("type", "object")
+            .endObject().endObject()
             .endObject().endObject().endObject().endObject());
         final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         try {
@@ -205,7 +209,8 @@ public void testObjectFieldNotAllowed() throws Exception {
 
     public void testNestedFieldNotAllowed() throws Exception {
         String mapping = Strings.toString(jsonBuilder().startObject().startObject("type").startObject("properties").startObject("my_field")
-            .field("type", "text").startObject("fields").startObject("multi").field("type", "nested").endObject().endObject()
+            .field("type", "text").startObject("fields").startObject("multi").field("type", "nested")
+            .endObject().endObject()
             .endObject().endObject().endObject().endObject());
         final DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         try {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index d590709d729a0..400426ee5710a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -51,7 +51,8 @@ public void testEmptyNested() throws Exception {
                 .startObject("nested1").field("type", "nested").endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
                 .bytes(XContentFactory.jsonBuilder()
@@ -79,7 +80,8 @@ public void testSingleNested() throws Exception {
                 .startObject("nested1").field("type", "nested").endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
@@ -130,7 +132,8 @@ public void testMultiNested() throws Exception {
                 .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
@@ -142,13 +145,21 @@ public void testMultiNested() throws Exception {
         assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(false));
         assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
-                        .startObject().field("field1", "1").startArray("nested2").startObject().field("field2", "2").endObject().startObject().field("field2", "3").endObject().endArray().endObject()
-                        .startObject().field("field1", "4").startArray("nested2").startObject().field("field2", "5").endObject().startObject().field("field2", "6").endObject().endArray().endObject()
+                            .startObject().field("field1", "1").startArray("nested2")
+                            .startObject().field("field2", "2").endObject()
+                            .startObject().field("field2", "3").endObject()
+                        .endArray()
+                        .endObject()
+                        .startObject().field("field1", "4")
+                        .startArray("nested2")
+                            .startObject().field("field2", "5").endObject()
+                            .startObject().field("field2", "6").endObject()
+                        .endArray().endObject()
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
@@ -182,7 +193,8 @@ public void testMultiObjectAndNested1() throws Exception {
                 .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
@@ -194,13 +206,21 @@ public void testMultiObjectAndNested1() throws Exception {
         assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(true));
         assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
-                        .startObject().field("field1", "1").startArray("nested2").startObject().field("field2", "2").endObject().startObject().field("field2", "3").endObject().endArray().endObject()
-                        .startObject().field("field1", "4").startArray("nested2").startObject().field("field2", "5").endObject().startObject().field("field2", "6").endObject().endArray().endObject()
+                        .startObject().field("field1", "1")
+                        .startArray("nested2")
+                            .startObject().field("field2", "2").endObject()
+                            .startObject().field("field2", "3").endObject()
+                        .endArray().endObject()
+                        .startObject().field("field1", "4")
+                        .startArray("nested2")
+                            .startObject().field("field2", "5").endObject()
+                            .startObject().field("field2", "6").endObject()
+                        .endArray().endObject()
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
@@ -228,13 +248,16 @@ public void testMultiObjectAndNested1() throws Exception {
     }
 
     public void testMultiObjectAndNested2() throws Exception {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("nested1").field("type", "nested").field("include_in_parent", true).startObject("properties")
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("nested1").field("type", "nested").field("include_in_parent", true)
+                .startObject("properties")
                 .startObject("nested2").field("type", "nested").field("include_in_parent", true)
                 .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
@@ -246,13 +269,21 @@ public void testMultiObjectAndNested2() throws Exception {
         assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(true));
         assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(false));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
-                        .startObject().field("field1", "1").startArray("nested2").startObject().field("field2", "2").endObject().startObject().field("field2", "3").endObject().endArray().endObject()
-                        .startObject().field("field1", "4").startArray("nested2").startObject().field("field2", "5").endObject().startObject().field("field2", "6").endObject().endArray().endObject()
+                        .startObject().field("field1", "1")
+                        .startArray("nested2")
+                            .startObject().field("field2", "2").endObject()
+                            .startObject().field("field2", "3").endObject()
+                        .endArray().endObject()
+                        .startObject().field("field1", "4")
+                        .startArray("nested2")
+                            .startObject().field("field2", "5").endObject()
+                            .startObject().field("field2", "6").endObject()
+                        .endArray().endObject()
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
@@ -286,7 +317,8 @@ public void testMultiRootAndNested1() throws Exception {
                 .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
@@ -298,13 +330,21 @@ public void testMultiRootAndNested1() throws Exception {
         assertThat(nested2Mapper.nested().isIncludeInParent(), equalTo(false));
         assertThat(nested2Mapper.nested().isIncludeInRoot(), equalTo(true));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
-                        .startObject().field("field1", "1").startArray("nested2").startObject().field("field2", "2").endObject().startObject().field("field2", "3").endObject().endArray().endObject()
-                        .startObject().field("field1", "4").startArray("nested2").startObject().field("field2", "5").endObject().startObject().field("field2", "6").endObject().endArray().endObject()
+                        .startObject().field("field1", "1")
+                        .startArray("nested2")
+                            .startObject().field("field2", "2").endObject()
+                            .startObject().field("field2", "3").endObject()
+                        .endArray().endObject()
+                        .startObject().field("field1", "4")
+                        .startArray("nested2")
+                            .startObject().field("field2", "5").endObject()
+                            .startObject().field("field2", "6").endObject()
+                        .endArray().endObject()
                         .endArray()
                         .endObject()),
                 XContentType.JSON));
@@ -339,15 +379,18 @@ public void testMultiRootAndNested1() throws Exception {
     public void testMultipleLevelsIncludeRoot1() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder()
             .startObject().startObject("type").startObject("properties")
-            .startObject("nested1").field("type", "nested").field("include_in_root", true).field("include_in_parent", true).startObject("properties")
-            .startObject("nested2").field("type", "nested").field("include_in_root", true).field("include_in_parent", true)
+            .startObject("nested1").field("type", "nested").field("include_in_root", true)
+            .field("include_in_parent", true).startObject("properties")
+            .startObject("nested2").field("type", "nested").field("include_in_root", true)
+            .field("include_in_parent", true)
             .endObject().endObject().endObject()
             .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject().startArray("nested1")
                         .startObject().startArray("nested2").startObject().field("foo", "bar")
                         .endObject().endArray().endObject().endArray()
@@ -377,10 +420,11 @@ public void testMultipleLevelsIncludeRoot2() throws Exception {
             .endObject().endObject().endObject().endObject().endObject()
             .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject().startArray("nested1")
                         .startObject().startArray("nested2")
                         .startObject().startArray("nested3").startObject().field("foo", "bar")
@@ -399,15 +443,16 @@ public void testNestedArrayStrict() throws Exception {
                 .endObject().endObject().endObject()
                 .endObject().endObject().endObject());
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         assertThat(docMapper.hasNestedObjects(), equalTo(true));
         ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
         assertThat(nested1Mapper.nested().isNested(), equalTo(true));
         assertThat(nested1Mapper.dynamic(), equalTo(Dynamic.STRICT));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
@@ -439,23 +484,30 @@ public void testLimitOfNestedFieldsPerIndex() throws Exception {
         };
 
         // default limit allows at least two nested fields
-        createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
+        createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")),
+            MergeReason.MAPPING_UPDATE, false);
 
         // explicitly setting limit to 0 prevents nested fields
         Exception e = expectThrows(IllegalArgumentException.class, () ->
-            createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0).build())
-                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false));
+            createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0)
+                .build())
+                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")),
+                MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), containsString("Limit of nested fields [0] in index [test2] has been exceeded"));
 
         // setting limit to 1 with 2 nested fields fails
         e = expectThrows(IllegalArgumentException.class, () ->
-            createIndex("test3", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 1).build())
-                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false));
+            createIndex("test3", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 1)
+                .build())
+                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")),
+                MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), containsString("Limit of nested fields [1] in index [test3] has been exceeded"));
 
         // do not check nested fields limit if mapping is not updated
-        createIndex("test4", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0).build())
-            .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_RECOVERY, false);
+        createIndex("test4", Settings.builder().put(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING.getKey(), 0)
+            .build())
+            .mapperService().merge("type", new CompressedXContent(mapping.apply("type")),
+            MergeReason.MAPPING_RECOVERY, false);
     }
 
     public void testParentObjectMapperAreNested() throws Exception {
@@ -511,8 +563,8 @@ public void testReorderParentBWC() throws IOException {
             ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
             assertThat(nested1Mapper.nested().isNested(), equalTo(true));
 
-            ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                    .bytes(XContentFactory.jsonBuilder()
+            ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1",
+                BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .field("field", "value")
                         .startArray("nested1")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java
index 815388eeffc55..550802f6c9dfd 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NullValueObjectMappingTests.java
@@ -36,10 +36,11 @@ public void testNullValueObject() throws IOException {
                 .startObject("properties").startObject("obj1").field("type", "object").endObject().endObject()
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        ParsedDocument doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("obj1").endObject()
                         .field("value1", "test1")
@@ -48,8 +49,8 @@ public void testNullValueObject() throws IOException {
 
         assertThat(doc.rootDoc().get("value1"), equalTo("test1"));
 
-        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .nullField("obj1")
                         .field("value1", "test1")
@@ -58,8 +59,8 @@ public void testNullValueObject() throws IOException {
 
         assertThat(doc.rootDoc().get("value1"), equalTo("test1"));
 
-        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
+        doc = defaultMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("obj1").field("field", "value").endObject()
                         .field("value1", "test1")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index 02d7cea2f4362..304d4b6aba373 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -67,7 +67,7 @@ public void doTestDefaults(String type) throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
         IndexableField dvField = fields[1];
@@ -118,7 +118,7 @@ public void doTestNoDocValues(String type) throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
     }
 
@@ -142,7 +142,7 @@ public void doTestStore(String type) throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(3, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
@@ -171,7 +171,7 @@ public void doTestCoerce(String type) throws IOException {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
@@ -345,7 +345,7 @@ protected void doTestNullValue(String type) throws IOException {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         assertEquals(123, pointField.numericValue().doubleValue(), 0d);
         IndexableField dvField = fields[1];
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
index 0df6f7a4fb78d..6a35e15ad867e 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java
@@ -40,7 +40,8 @@ public void testDifferentInnerObjectTokenFailure() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject());
 
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
             defaultMapper.parse(SourceToParse.source("test", "type", "1", new BytesArray(" {\n" +
                 "      \"object\": {\n" +
@@ -68,38 +69,37 @@ public void testEmptyArrayProperties() throws Exception {
     }
 
     public void testEmptyFieldsArrayMultiFields() throws Exception {
-        String mapping = Strings
-                                        .toString(XContentFactory.jsonBuilder()
-                                                                        .startObject()
-                                                                            .startObject("tweet")
-                                                                                .startObject("properties")
-                                                                                    .startObject("name")
-                                                                                        .field("type", "text")
-                                                                                        .startArray("fields")
-                                                                                        .endArray()
-                                                                                    .endObject()
-                                                                                .endObject()
-                                                                            .endObject()
-                                                                        .endObject());
+        String mapping =
+            Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("tweet")
+                        .startObject("properties")
+                            .startObject("name")
+                                .field("type", "text")
+                                .startArray("fields")
+                                .endArray()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject());
         createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping));
     }
 
     public void testFieldsArrayMultiFieldsShouldThrowException() throws Exception {
-        String mapping = Strings
-                .toString(XContentFactory.jsonBuilder()
-                        .startObject()
-                            .startObject("tweet")
-                                .startObject("properties")
-                                    .startObject("name")
-                                        .field("type", "text")
-                                        .startArray("fields")
-                                            .startObject().field("test", "string").endObject()
-                                            .startObject().field("test2", "string").endObject()
-                                        .endArray()
-                                    .endObject()
-                                .endObject()
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("tweet")
+                        .startObject("properties")
+                            .startObject("name")
+                                .field("type", "text")
+                                .startArray("fields")
+                                    .startObject().field("test", "string").endObject()
+                                    .startObject().field("test2", "string").endObject()
+                                .endArray()
                             .endObject()
-                        .endObject());
+                        .endObject()
+                    .endObject()
+                .endObject());
         try {
             createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping));
             fail("Expected MapperParsingException");
@@ -110,32 +110,30 @@ public void testFieldsArrayMultiFieldsShouldThrowException() throws Exception {
     }
 
     public void testEmptyFieldsArray() throws Exception {
-        String mapping = Strings
-                                        .toString(XContentFactory.jsonBuilder()
-                                                                        .startObject()
-                                                                            .startObject("tweet")
-                                                                                .startObject("properties")
-                                                                                    .startArray("fields")
-                                                                                    .endArray()
-                                                                                .endObject()
-                                                                            .endObject()
-                                                                        .endObject());
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("tweet")
+                        .startObject("properties")
+                            .startArray("fields")
+                            .endArray()
+                        .endObject()
+                    .endObject()
+                .endObject());
         createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping));
     }
 
     public void testFieldsWithFilledArrayShouldThrowException() throws Exception {
-        String mapping = Strings
-                .toString(XContentFactory.jsonBuilder()
-                        .startObject()
-                            .startObject("tweet")
-                                .startObject("properties")
-                                    .startArray("fields")
-                                        .startObject().field("test", "string").endObject()
-                                        .startObject().field("test2", "string").endObject()
-                                    .endArray()
-                                .endObject()
-                            .endObject()
-                        .endObject());
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("tweet")
+                        .startObject("properties")
+                            .startArray("fields")
+                                .startObject().field("test", "string").endObject()
+                                .startObject().field("test2", "string").endObject()
+                            .endArray()
+                        .endObject()
+                    .endObject()
+                .endObject());
         try {
             createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping));
             fail("Expected MapperParsingException");
@@ -145,22 +143,21 @@ public void testFieldsWithFilledArrayShouldThrowException() throws Exception {
     }
 
     public void testFieldPropertiesArray() throws Exception {
-        String mapping = Strings
-                                        .toString(XContentFactory.jsonBuilder()
-                                                                        .startObject()
-                                                                            .startObject("tweet")
-                                                                                .startObject("properties")
-                                                                                    .startObject("name")
-                                                                                        .field("type", "text")
-                                                                                        .startObject("fields")
-                                                                                            .startObject("raw")
-                                                                                                .field("type", "keyword")
-                                                                                            .endObject()
-                                                                                        .endObject()
-                                                                                    .endObject()
-                                                                                .endObject()
-                                                                            .endObject()
-                                                                        .endObject());
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("tweet")
+                        .startObject("properties")
+                            .startObject("name")
+                                .field("type", "text")
+                                .startObject("fields")
+                                    .startObject("raw")
+                                        .field("type", "keyword")
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject());
         createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping));
     }
 
@@ -188,7 +185,8 @@ public void testMerge() throws IOException {
     }
 
     public void testEmptyName() throws Exception {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
                 .startObject("")
                     .startObject("properties")
                         .startObject("name")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
index aa9589d70dca0..4b641023d56ec 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
@@ -64,7 +64,8 @@ public void testParentSetInDocNotAllowed() throws Exception {
                 .startObject().field("_parent", "1122").endObject()), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
-            assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
+            assertThat(e.getMessage(), e.getMessage(),
+                containsString("Field [_parent] is a metadata field and cannot be added inside a document"));
         }
     }
 
@@ -94,7 +95,8 @@ public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
             .startObject("properties")
             .endObject()
             .endObject().endObject();
-        mapperService.merge("some_type", new CompressedXContent(Strings.toString(mappingSource)), MergeReason.MAPPING_UPDATE, false);
+        mapperService.merge("some_type", new CompressedXContent(Strings.toString(mappingSource)),
+            MergeReason.MAPPING_UPDATE, false);
         Set<String> allFields = new HashSet<>(mapperService.simpleMatchToFullName("*"));
         assertTrue(allFields.contains("_parent"));
         assertFalse(allFields.contains("_parent#null"));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
index 6bb15432b1fd3..c96b8c44ee707 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/PathMapperTests.java
@@ -31,7 +31,8 @@
 public class PathMapperTests extends ESSingleNodeTestCase {
     public void testPathMapping() throws IOException {
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/path/test-mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("person", new CompressedXContent(mapping));
 
         // test full name
         assertThat(docMapper.mappers().getMapper("first1"), nullValue());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
index 00068f76e753d..1f8b0b58af813 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -136,7 +136,7 @@ public void doTestDefaults(String type) throws Exception {
         assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
 
         IndexableField pointField = fields[1];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
     }
 
@@ -188,7 +188,7 @@ protected void doTestNoDocValues(String type) throws Exception {
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.length);
         IndexableField pointField = fields[0];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
     }
 
     @Override
@@ -216,7 +216,7 @@ protected void doTestStore(String type) throws Exception {
         IndexableField dvField = fields[0];
         assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
         IndexableField pointField = fields[1];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
         IndexableField storedField = fields[2];
         assertTrue(storedField.fieldType().stored());
         String strVal = "5";
@@ -255,7 +255,7 @@ public void doTestCoerce(String type) throws IOException {
         IndexableField dvField = fields[0];
         assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
         IndexableField pointField = fields[1];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
 
         // date_range ignores the coerce parameter and epoch_millis date format truncates floats (see issue: #14641)
         if (type.equals("date_range") == false) {
@@ -353,7 +353,7 @@ protected void doTestNullValue(String type) throws IOException {
         IndexableField dvField = fields[0];
         assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
         IndexableField pointField = fields[1];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         storedField = fields[2];
         assertTrue(storedField.fieldType().stored());
@@ -406,7 +406,7 @@ public void doTestNoBounds(String type) throws IOException {
         IndexableField dvField = fields[0];
         assertEquals(DocValuesType.BINARY, dvField.fieldType().docValuesType());
         IndexableField pointField = fields[1];
-        assertEquals(2, pointField.fieldType().pointDimensionCount());
+        assertEquals(2, pointField.fieldType().pointIndexDimensionCount());
         assertFalse(pointField.fieldType().stored());
         IndexableField storedField = fields[2];
         assertTrue(storedField.fieldType().stored());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
index 1b83b1bcb5b67..11c858545d03f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RoutingFieldMapperTests extends ESSingleNodeTestCase {
@@ -33,7 +34,8 @@ public class RoutingFieldMapperTests extends ESSingleNodeTestCase {
     public void testRoutingMapper() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference
             .bytes(XContentFactory.jsonBuilder()
@@ -48,14 +50,16 @@ public void testRoutingMapper() throws Exception {
 
     public void testIncludeInObjectNotAllowed() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject().field("_routing", "foo").endObject()),XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
-            assertTrue(e.getMessage(), e.getMessage().contains("Field [_routing] is a metadata field and cannot be added inside a document"));
+            assertThat(e.getMessage(), e.getMessage(),
+                containsString("Field [_routing] is a metadata field and cannot be added inside a document"));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index 7266960508398..f364d1e37761f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -46,7 +46,8 @@ public void testNoFormat() throws Exception {
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(mapping));
-        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
                 .field("field", "value")
                 .endObject()),
                 XContentType.JSON));
@@ -54,7 +55,8 @@ public void testNoFormat() throws Exception {
         assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON));
 
         documentMapper = parser.parse("type", new CompressedXContent(mapping));
-        doc = documentMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.smileBuilder().startObject()
+        doc = documentMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.smileBuilder().startObject()
                 .field("field", "value")
                 .endObject()),
                 XContentType.SMILE));
@@ -67,9 +69,11 @@ public void testIncludes() throws Exception {
             .startObject("_source").array("includes", new String[]{"path1*"}).endObject()
             .endObject().endObject());
 
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
             .startObject("path1").field("field1", "value1").endObject()
             .startObject("path2").field("field2", "value2").endObject()
             .endObject()),
@@ -89,9 +93,11 @@ public void testExcludes() throws Exception {
             .startObject("_source").array("excludes", new String[]{"path1*"}).endObject()
             .endObject().endObject());
 
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+        ParsedDocument doc = documentMapper.parse(SourceToParse.source("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
             .startObject("path1").field("field1", "value1").endObject()
             .startObject("path2").field("field2", "value2").endObject()
             .endObject()),
@@ -241,10 +247,12 @@ public void testComplete() throws Exception {
 
     public void testSourceObjectContainsExtraTokens() throws Exception {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject());
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("type", new CompressedXContent(mapping));
 
         try {
-            documentMapper.parse(SourceToParse.source("test", "type", "1", new BytesArray("{}}"), XContentType.JSON)); // extra end object (invalid JSON)
+            documentMapper.parse(SourceToParse.source("test", "type", "1",
+                new BytesArray("{}}"), XContentType.JSON)); // extra end object (invalid JSON)
             fail("Expected parse exception");
         } catch (MapperParsingException e) {
             assertNotNull(e.getRootCause());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
index 338e5e4d5708d..15fe675d34ea2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
@@ -42,7 +42,7 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
-    protected void testConflict(String mapping, String mappingUpdate, Version idxVersion, String... errorMessages) throws InterruptedException {
+    protected void testConflict(String mapping, String mappingUpdate, Version idxVersion,String... errorMessages) {
         assertAcked(prepareCreate(INDEX).setSource(mapping, XContentType.JSON)
             .setSettings(Settings.builder().put("index.version.created", idxVersion.id)));
         ensureGreen(INDEX);
@@ -101,7 +101,8 @@ private void compareMappingOnNodes(GetMappingsResponse previousMapping) {
         // make sure all nodes have same cluster state
         for (Client client : cluster().getClients()) {
             GetMappingsResponse currentMapping = client.admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).setLocal(true).get();
-            assertThat(previousMapping.getMappings().get(INDEX).get(TYPE).source(), equalTo(currentMapping.getMappings().get(INDEX).get(TYPE).source()));
+            assertThat(previousMapping.getMappings().get(INDEX).get(TYPE).source(),
+                equalTo(currentMapping.getMappings().get(INDEX).get(TYPE).source()));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
index cdaa7ad8ba3cd..b9eb2e0c1bce5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
@@ -72,7 +72,8 @@ protected void testConflictWhileMergingAndMappingUnchanged(XContentBuilder mappi
         CompressedXContent mappingBeforeUpdate = indexService.mapperService().documentMapper("type").mappingSource();
         // simulate like in MetaDataMappingService#putMapping
         try {
-            indexService.mapperService().merge("type", new CompressedXContent(BytesReference.bytes(mappingUpdate)), MapperService.MergeReason.MAPPING_UPDATE, false);
+            indexService.mapperService().merge("type", new CompressedXContent(BytesReference.bytes(mappingUpdate)),
+                MapperService.MergeReason.MAPPING_UPDATE, false);
             fail();
         } catch (IllegalArgumentException e) {
             // expected
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index c2b4183cd5050..f1bccfca7866d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -456,7 +456,7 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
         DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
         query.from(queryFromValue);
         query.to(queryToValue);
-        query.timeZone(randomFrom(DateTimeZone.getAvailableIDs()));
+        query.timeZone(randomDateTimeZone().getID());
         query.format("yyyy-MM-dd");
         QueryShardContext queryShardContext = createShardContext();
         QueryBuilder rewritten = query.rewrite(queryShardContext);
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 7bd80c7ef80bb..5275853af3a42 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -105,6 +105,12 @@ protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
                 fields.put(STRING_FIELD_NAME_2, 2.0f / randomIntBetween(1, 20));
             }
         }
+        // special handling if query is "now" and no field specified. This hits the "mapped_date" field which leads to the query not being
+        // cacheable and trigger later test failures (see https://github.com/elastic/elasticsearch/issues/35183)
+        if (fieldCount == 0 && result.value().equalsIgnoreCase("now")) {
+            fields.put(STRING_FIELD_NAME_2, 2.0f / randomIntBetween(1, 20));
+        }
+
         result.fields(fields);
         if (randomBoolean()) {
             result.autoGenerateSynonymsPhraseQuery(randomBoolean());
diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 363ff6a578bd9..aa11b77a804df 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query.functionscore;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -696,6 +695,29 @@ public void testRewriteWithFunction() throws IOException {
         assertSame(rewrite.filterFunctionBuilders()[1].getFilter(), secondFunction);
     }
 
+    /**
+     * Please see https://github.com/elastic/elasticsearch/issues/35123 for context.
+     */
+    public void testSingleScriptFunction() throws IOException {
+        QueryBuilder queryBuilder = RandomQueryBuilder.createQuery(random());
+        ScoreFunctionBuilder functionBuilder = new ScriptScoreFunctionBuilder(
+            new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap()));
+
+        FunctionScoreQueryBuilder builder = functionScoreQuery(queryBuilder, functionBuilder);
+        if (randomBoolean()) {
+            builder.boostMode(randomFrom(CombineFunction.values()));
+        }
+
+        Query query = builder.toQuery(createShardContext());
+        assertThat(query, instanceOf(FunctionScoreQuery.class));
+
+        CombineFunction expectedBoostMode = builder.boostMode() != null
+            ? builder.boostMode()
+            : FunctionScoreQueryBuilder.DEFAULT_BOOST_MODE;
+        CombineFunction actualBoostMode = ((FunctionScoreQuery) query).getCombineFunction();
+        assertEquals(expectedBoostMode, actualBoostMode);
+    }
+
     public void testQueryMalformedArrayNotSupported() throws IOException {
         String json =
             "{\n" +
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index 0888dfd3c40c5..7b430bd803c11 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -60,7 +60,7 @@ public void setUp() throws Exception {
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,  boundAddress -> clusterService.localNode(), null, Collections.emptySet());
         transportService.start();
         transportService.acceptIncomingRequests();
-        shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
+        shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
     }
 
     public void tearDown() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/SequenceNumbersTests.java b/server/src/test/java/org/elasticsearch/index/seqno/SequenceNumbersTests.java
index f835cff3f4656..9a279feddb973 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/SequenceNumbersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/SequenceNumbersTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.seqno;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -57,4 +58,11 @@ public void testMax() {
         assertThat(e, hasToString(containsString("sequence number must be assigned")));
     }
 
+    public void testSeqNoStatsEqualsAndHashCode() {
+        final long maxSeqNo = randomLongBetween(SequenceNumbers.UNASSIGNED_SEQ_NO, Long.MAX_VALUE);
+        final long localCheckpoint = randomLongBetween(SequenceNumbers.UNASSIGNED_SEQ_NO, maxSeqNo);
+        final long globalCheckpoint = randomLongBetween(SequenceNumbers.UNASSIGNED_SEQ_NO, localCheckpoint);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(new SeqNoStats(maxSeqNo, localCheckpoint, globalCheckpoint),
+            stats -> new SeqNoStats(stats.getMaxSeqNo(), stats.getLocalCheckpoint(), stats.getGlobalCheckpoint()));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 96058d1fdac75..3d63efcb05e6c 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -928,6 +928,7 @@ public void testRestoreLocalHistoryFromTranslogOnPromotion() throws IOException,
         final long maxSeqNoOfUpdatesOrDeletes = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, maxSeqNo);
         final Set<String> docsBeforeRollback = getShardDocUIDs(indexShard);
         final CountDownLatch latch = new CountDownLatch(1);
+        final boolean shouldRollback = Math.max(globalCheckpointOnReplica, globalCheckpoint) < maxSeqNo;
         indexShard.acquireReplicaOperationPermit(
                 indexShard.getPendingPrimaryTerm() + 1,
                 globalCheckpoint,
@@ -947,10 +948,13 @@ public void onFailure(Exception e) {
                 ThreadPool.Names.SAME, "");
 
         latch.await();
-        assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Collections.max(
-            Arrays.asList(maxSeqNoOfUpdatesOrDeletes, globalCheckpoint, globalCheckpointOnReplica))
-        ));
-
+        if (shouldRollback) {
+            assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Collections.max(
+                Arrays.asList(maxSeqNoOfUpdatesOrDeletes, globalCheckpoint, globalCheckpointOnReplica))
+            ));
+        } else {
+            assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(maxSeqNoOfUpdatesOrDeletes, currentMaxSeqNoOfUpdates)));
+        }
         final ShardRouting newRouting = indexShard.routingEntry().moveActiveReplicaToPrimary();
         final CountDownLatch resyncLatch = new CountDownLatch(1);
         indexShard.updateShardState(
@@ -965,9 +969,13 @@ public void onFailure(Exception e) {
         assertThat(indexShard.getLocalCheckpoint(), equalTo(maxSeqNo));
         assertThat(indexShard.seqNoStats().getMaxSeqNo(), equalTo(maxSeqNo));
         assertThat(getShardDocUIDs(indexShard), equalTo(docsBeforeRollback));
-        assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Collections.max(
-            Arrays.asList(currentMaxSeqNoOfUpdates, maxSeqNoOfUpdatesOrDeletes, globalCheckpoint, globalCheckpointOnReplica))
-        ));
+        if (shouldRollback) {
+            assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Collections.max(
+                Arrays.asList(currentMaxSeqNoOfUpdates, maxSeqNoOfUpdatesOrDeletes, globalCheckpoint, globalCheckpointOnReplica))
+            ));
+        } else {
+            assertThat(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(currentMaxSeqNoOfUpdates, maxSeqNoOfUpdatesOrDeletes)));
+        }
         closeShard(indexShard, false);
     }
 
@@ -1050,7 +1058,7 @@ public void testConcurrentTermIncreaseOnReplicaShard() throws BrokenBarrierExcep
                         @Override
                         public void onResponse(Releasable releasable) {
                             counter.incrementAndGet();
-                            assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + increment));
+                            assertThat(indexShard.getOperationPrimaryTerm(), equalTo(primaryTerm + increment));
                             latch.countDown();
                             releasable.close();
                         }
@@ -1095,6 +1103,7 @@ public void onFailure(Exception e) {
         }
 
         assertThat(indexShard.getPendingPrimaryTerm(), equalTo(primaryTerm + Math.max(firstIncrement, secondIncrement)));
+        assertThat(indexShard.getOperationPrimaryTerm(), equalTo(indexShard.getPendingPrimaryTerm()));
 
         closeShards(indexShard);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
index 28e625b34dfd6..62f6a7b234588 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -68,7 +68,7 @@ public void testSyncerSendsOffCorrectDocuments() throws Exception {
                 assertThat(parentTask, instanceOf(PrimaryReplicaSyncer.ResyncTask.class));
                 listener.onResponse(new ResyncReplicationResponse());
             };
-        PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(Settings.EMPTY, taskManager, syncAction);
+        PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(taskManager, syncAction);
         syncer.setChunkSize(new ByteSizeValue(randomIntBetween(1, 10)));
 
         int numDocs = randomInt(10);
@@ -136,7 +136,7 @@ public void testSyncerOnClosingShard() throws Exception {
                 syncActionCalled.set(true);
                 threadPool.generic().execute(() -> listener.onResponse(new ResyncReplicationResponse()));
             };
-        PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(Settings.EMPTY,
+        PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(
             new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), syncAction);
         syncer.setChunkSize(new ByteSizeValue(1)); // every document is sent off separately
 
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 677c95a85cafb..91fd9649c92e8 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -107,6 +107,9 @@
 import static org.mockito.Mockito.when;
 
 public class ClusterStateChanges extends AbstractComponent {
+    private static final Settings SETTINGS = Settings.builder()
+            .put(PATH_HOME_SETTING.getKey(), "dummy")
+            .build();
 
     private final AllocationService allocationService;
     private final ClusterService clusterService;
@@ -125,21 +128,19 @@ public class ClusterStateChanges extends AbstractComponent {
     private final NodeJoinController.JoinTaskExecutor joinTaskExecutor;
 
     public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool threadPool) {
-        super(Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build());
-
-        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        allocationService = new AllocationService(settings, new AllocationDeciders(settings,
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(settings, clusterSettings),
-                new ReplicaAfterPrimaryActiveAllocationDecider(settings),
+        ClusterSettings clusterSettings = new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        allocationService = new AllocationService(new AllocationDeciders(
+            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(SETTINGS, clusterSettings),
+                new ReplicaAfterPrimaryActiveAllocationDecider(),
                 new RandomAllocationDeciderTests.RandomAllocationDecider(getRandom())))),
-            new TestGatewayAllocator(), new BalancedShardsAllocator(settings),
+            new TestGatewayAllocator(), new BalancedShardsAllocator(SETTINGS),
             EmptyClusterInfoService.INSTANCE);
         shardFailedClusterStateTaskExecutor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocationService, null, logger);
         shardStartedClusterStateTaskExecutor = new ShardStateAction.ShardStartedClusterStateTaskExecutor(allocationService, logger);
         ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
-        DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterSettings);
-        Environment environment = TestEnvironment.newEnvironment(settings);
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(SETTINGS);
+        DestructiveOperations destructiveOperations = new DestructiveOperations(SETTINGS, clusterSettings);
+        Environment environment = TestEnvironment.newEnvironment(SETTINGS);
         Transport transport = mock(Transport.class); // it's not used
 
         // mocks
@@ -166,11 +167,11 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th
         }
 
         // services
-        TransportService transportService = new TransportService(settings, transport, threadPool,
+        TransportService transportService = new TransportService(SETTINGS, transport, threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,
-            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings,
+            boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings,
             Collections.emptySet());
-        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null,
+        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(SETTINGS, xContentRegistry, null, null,
             null) {
             // metaData upgrader should do nothing
             @Override
@@ -178,29 +179,29 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m
                 return indexMetaData;
             }
         };
-        MetaDataIndexStateService indexStateService = new MetaDataIndexStateService(settings, clusterService, allocationService,
+        MetaDataIndexStateService indexStateService = new MetaDataIndexStateService(clusterService, allocationService,
             metaDataIndexUpgradeService, indicesService, threadPool);
-        MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService);
-        MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
+        MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(SETTINGS, clusterService, allocationService);
+        MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(clusterService,
             allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, threadPool);
-        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
-            allocationService, new AliasValidator(settings), environment,
+        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(SETTINGS, clusterService, indicesService,
+            allocationService, new AliasValidator(), environment,
             IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, true);
 
-        transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool,
+        transportCloseIndexAction = new TransportCloseIndexAction(SETTINGS, transportService, clusterService, threadPool,
             indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations);
-        transportOpenIndexAction = new TransportOpenIndexAction(settings, transportService,
+        transportOpenIndexAction = new TransportOpenIndexAction(SETTINGS, transportService,
             clusterService, threadPool, indexStateService, actionFilters, indexNameExpressionResolver, destructiveOperations);
-        transportDeleteIndexAction = new TransportDeleteIndexAction(settings, transportService,
+        transportDeleteIndexAction = new TransportDeleteIndexAction(SETTINGS, transportService,
             clusterService, threadPool, deleteIndexService, actionFilters, indexNameExpressionResolver, destructiveOperations);
-        transportUpdateSettingsAction = new TransportUpdateSettingsAction(settings,
+        transportUpdateSettingsAction = new TransportUpdateSettingsAction(SETTINGS,
             transportService, clusterService, threadPool, metaDataUpdateSettingsService, actionFilters, indexNameExpressionResolver);
-        transportClusterRerouteAction = new TransportClusterRerouteAction(settings,
+        transportClusterRerouteAction = new TransportClusterRerouteAction(SETTINGS,
             transportService, clusterService, threadPool, allocationService, actionFilters, indexNameExpressionResolver);
-        transportCreateIndexAction = new TransportCreateIndexAction(settings,
+        transportCreateIndexAction = new TransportCreateIndexAction(SETTINGS,
             transportService, clusterService, threadPool, createIndexService, actionFilters, indexNameExpressionResolver);
 
-        ElectMasterService electMasterService = new ElectMasterService(settings);
+        ElectMasterService electMasterService = new ElectMasterService(SETTINGS);
         nodeRemovalExecutor = new ZenDiscovery.NodeRemovalClusterStateTaskExecutor(allocationService, electMasterService,
             s -> { throw new AssertionError("rejoin not implemented"); }, logger);
         joinTaskExecutor = new NodeJoinController.JoinTaskExecutor(allocationService, electMasterService, logger);
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 39091ce04ec6e..4625aa04be372 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -462,7 +462,7 @@ private IndicesClusterStateService createIndicesClusterStateService(DiscoveryNod
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(settings, clusterService,
             transportService, null, threadPool);
-        final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(settings, threadPool,
+        final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService(threadPool,
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
         final PrimaryReplicaSyncer primaryReplicaSyncer = mock(PrimaryReplicaSyncer.class);
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java
index 524795bfa2480..72eb2baeca942 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoverySourceServiceTests.java
@@ -37,7 +37,7 @@ public class PeerRecoverySourceServiceTests extends IndexShardTestCase {
 
     public void testDuplicateRecoveries() throws IOException {
         IndexShard primary = newStartedShard(true);
-        PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(Settings.EMPTY,
+        PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(
             mock(TransportService.class), mock(IndicesService.class),
             new RecoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(primary.shardId(), randomAlphaOfLength(10),
diff --git a/server/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
index 234524f16f454..873ebf17f2382 100644
--- a/server/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
@@ -21,7 +21,6 @@
 
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeEnvironment.NodePath;
 import org.elasticsearch.test.ESTestCase;
@@ -51,7 +50,7 @@ public class FsProbeTests extends ESTestCase {
     public void testFsInfo() throws IOException {
 
         try (NodeEnvironment env = newNodeEnvironment()) {
-            FsProbe probe = new FsProbe(Settings.EMPTY, env);
+            FsProbe probe = new FsProbe(env);
 
             FsInfo stats = probe.stats(null, null);
             assertNotNull(stats);
@@ -166,7 +165,7 @@ public void testIoStats() {
                 " 253       1 dm-1 112 0 4624 13 0 0 0 0 0 5 13",
                 " 253       2 dm-2 47802 0 710658 49312 1371977 0 64126096 33730596 0 1058193 33781827"));
 
-        final FsProbe probe = new FsProbe(Settings.EMPTY, null) {
+        final FsProbe probe = new FsProbe(null) {
             @Override
             List<String> readProcDiskStats() throws IOException {
                 return diskStats.get();
diff --git a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
index d86d7b46cc7a1..f6bb9d8375845 100644
--- a/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/node/ResponseCollectorServiceTests.java
@@ -54,7 +54,7 @@ public void setUp() throws Exception {
         clusterService = new ClusterService(Settings.EMPTY,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 threadpool, Collections.emptyMap());
-        collector = new ResponseCollectorService(Settings.EMPTY, clusterService);
+        collector = new ResponseCollectorService(clusterService);
     }
 
     @After
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
index f13a35613d530..3eec748808ecf 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
@@ -640,8 +640,8 @@ private void changeRoutingTable(MetaData.Builder metaData, RoutingTable.Builder
 
     /** Creates a PersistentTasksClusterService with a single PersistentTasksExecutor implemented by a BiFunction **/
     private <P extends PersistentTaskParams> PersistentTasksClusterService createService(final BiFunction<P, ClusterState, Assignment> fn) {
-        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY,
-            singleton(new PersistentTasksExecutor<P>(Settings.EMPTY, TestPersistentTasksExecutor.NAME, null) {
+        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(
+            singleton(new PersistentTasksExecutor<P>(TestPersistentTasksExecutor.NAME, null) {
                 @Override
                 public Assignment getAssignment(P params, ClusterState clusterState) {
                     return fn.apply(params, clusterState);
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java
index 655a21a5f5390..7f2dada7c4cb7 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksDecidersTestCase.java
@@ -58,10 +58,10 @@ public static void setUpThreadPool() {
     public void setUp() throws Exception {
         super.setUp();
         clusterService = createClusterService(threadPool);
-        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(clusterService.getSettings(), emptyList()) {
+        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(emptyList()) {
             @Override
             public <Params extends PersistentTaskParams> PersistentTasksExecutor<Params> getPersistentTaskExecutorSafe(String taskName) {
-                return new PersistentTasksExecutor<Params>(clusterService.getSettings(), taskName, null) {
+                return new PersistentTasksExecutor<Params>(taskName, null) {
                     @Override
                     protected void nodeOperation(AllocatedPersistentTask task, Params params, PersistentTaskState state) {
                         logger.debug("Executing task {}", task);
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java
index 906ecf232053d..4ddee816e7f71 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceTests.java
@@ -106,10 +106,10 @@ public void testStartTask() {
             when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn(
                     new TestPersistentTasksPlugin.TestTask(i, "persistent", "test", "", parentId, Collections.emptyMap()));
         }
-        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
+        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Collections.singletonList(action));
 
         MockExecutor executor = new MockExecutor();
-        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
+        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(persistentTasksService,
                 registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor);
 
         ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY);
@@ -202,10 +202,10 @@ public void testParamsStatusAndNodeTaskAreDelegated() throws Exception {
         AllocatedPersistentTask nodeTask =
                 new TestPersistentTasksPlugin.TestTask(0, "persistent", "test", "", parentId, Collections.emptyMap());
         when(action.createTask(anyLong(), anyString(), anyString(), eq(parentId), any(), any())).thenReturn(nodeTask);
-        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
+        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Collections.singletonList(action));
 
         MockExecutor executor = new MockExecutor();
-        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
+        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(persistentTasksService,
                 registry, new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()), executor);
 
         ClusterState state = createInitialClusterState(1, Settings.EMPTY);
@@ -231,7 +231,7 @@ public void testParamsStatusAndNodeTaskAreDelegated() throws Exception {
     public void testTaskCancellation() {
         AtomicLong capturedTaskId = new AtomicLong();
         AtomicReference<ActionListener<CancelTasksResponse>> capturedListener = new AtomicReference<>();
-        PersistentTasksService persistentTasksService = new PersistentTasksService(Settings.EMPTY, null, null, null) {
+        PersistentTasksService persistentTasksService = new PersistentTasksService(null, null, null) {
             @Override
             void sendCancelRequest(final long taskId, final String reason, final ActionListener<CancelTasksResponse> listener) {
                 capturedTaskId.set(taskId);
@@ -250,12 +250,12 @@ public void sendCompletionRequest(final String taskId, final long taskAllocation
         when(action.createTask(anyLong(), anyString(), anyString(), any(), any(), any()))
                 .thenReturn(new TestPersistentTasksPlugin.TestTask(1, "persistent", "test", "", new TaskId("cluster", 1),
                         Collections.emptyMap()));
-        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Settings.EMPTY, Collections.singletonList(action));
+        PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry(Collections.singletonList(action));
 
         int nonLocalNodesCount = randomInt(10);
         MockExecutor executor = new MockExecutor();
         TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet());
-        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(Settings.EMPTY, persistentTasksService,
+        PersistentTasksNodeService coordinator = new PersistentTasksNodeService(persistentTasksService,
                 registry, taskManager, executor);
 
         ClusterState state = createInitialClusterState(nonLocalNodesCount, Settings.EMPTY);
diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
index cfcc5ed8c0f63..fdf1c8d39852a 100644
--- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
+++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
@@ -92,7 +92,7 @@ public class TestPersistentTasksPlugin extends Plugin implements ActionPlugin, P
     @Override
     public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterService clusterService,
                                                                        ThreadPool threadPool, Client client) {
-        return Collections.singletonList(new TestPersistentTasksExecutor(Settings.EMPTY, clusterService));
+        return Collections.singletonList(new TestPersistentTasksExecutor(clusterService));
     }
 
     @Override
@@ -293,8 +293,8 @@ public static class TestPersistentTasksExecutor extends PersistentTasksExecutor<
         public static final String NAME = "cluster:admin/persistent/test";
         private final ClusterService clusterService;
 
-        public TestPersistentTasksExecutor(Settings settings, ClusterService clusterService) {
-            super(settings, NAME, ThreadPool.Names.GENERIC);
+        public TestPersistentTasksExecutor(ClusterService clusterService) {
+            super(NAME, ThreadPool.Names.GENERIC);
             this.clusterService = clusterService;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index f36638a43909f..4aca4cc4e6d0b 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
-import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -83,12 +82,12 @@ public void setup() {
                 .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), BREAKER_LIMIT)
                 .build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        usageService = new UsageService(settings);
+        usageService = new UsageService();
         // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
         inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
 
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        restController = new RestController(settings, Collections.emptySet(), null, null, circuitBreakerService, usageService);
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (request, channel, client) -> channel.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -102,7 +101,7 @@ public void setup() {
     public void testApplyRelevantHeaders() throws Exception {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<String> headers = new HashSet<>(Arrays.asList("header.1", "header.2"));
-        final RestController restController = new RestController(Settings.EMPTY, headers, null, null, circuitBreakerService, usageService);
+        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("true"));
         restHeaders.put("header.2", Collections.singletonList("true"));
@@ -135,8 +134,7 @@ public MethodHandlers next() {
     }
 
     public void testCanTripCircuitBreaker() throws Exception {
-        RestController controller = new RestController(Settings.EMPTY, Collections.emptySet(), null, null, circuitBreakerService,
-                usageService);
+        RestController controller = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         // trip circuit breaker by default
         controller.registerHandler(RestRequest.Method.GET, "/trip", new FakeRestHandler(true));
         controller.registerHandler(RestRequest.Method.GET, "/do-not-trip", new FakeRestHandler(false));
@@ -208,7 +206,7 @@ public void testRestHandlerWrapper() throws Exception {
             assertSame(handler, h);
             return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
         };
-        final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper, null,
+        final RestController restController = new RestController(Collections.emptySet(), wrapper, null,
                 circuitBreakerService, usageService);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, Optional.of(handler));
@@ -290,9 +288,7 @@ public void testDispatchRequiresContentTypeForRequestsWithContent() {
         String content = randomAlphaOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
         TestRestRequest request = new TestRestRequest("/", content, null);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
-        restController = new RestController(
-            Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(),
-                Collections.emptySet(), null, null, circuitBreakerService, usageService);
+        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (r, c, client) -> c.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
diff --git a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
index ebe8ae00ac0f3..47c277bccc5c7 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestHttpResponseHeadersTests.java
@@ -88,8 +88,8 @@ public void testUnsupportedMethodResponseHttpHeader() throws Exception {
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
 
         final Settings settings = Settings.EMPTY;
-        UsageService usageService = new UsageService(settings);
-        RestController restController = new RestController(settings, Collections.emptySet(),
+        UsageService usageService = new UsageService();
+        RestController restController = new RestController(Collections.emptySet(),
                 null, null, circuitBreakerService, usageService);
 
         // A basic RestHandler handles requests to the endpoint
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsActionTests.java
index 29b19739e7587..79d04e2816117 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsActionTests.java
@@ -52,7 +52,7 @@ private void runTestFilterSettingsTest(
         final Settings settings = Settings.builder().put("foo.filtered", "bar").put("foo.non_filtered", "baz").build();
         md.accept(mdBuilder, settings);
         final ClusterState.Builder builder = new ClusterState.Builder(ClusterState.EMPTY_STATE).metaData(mdBuilder);
-        final SettingsFilter filter = new SettingsFilter(Settings.EMPTY, Collections.singleton("foo.filtered"));
+        final SettingsFilter filter = new SettingsFilter(Collections.singleton("foo.filtered"));
         final Setting.Property[] properties = {Setting.Property.Dynamic, Setting.Property.Filtered, Setting.Property.NodeScope};
         final Set<Setting<?>> settingsSet = Stream.concat(
                 ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(),
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
index 90e0cfa656b7e..1acdb8c5d6211 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
@@ -48,9 +48,9 @@ public class RestNodesStatsActionTests extends ESTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        UsageService usageService = new UsageService(Settings.EMPTY);
+        UsageService usageService = new UsageService();
         action = new RestNodesStatsAction(Settings.EMPTY,
-                new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null, usageService));
+                new RestController(Collections.emptySet(), null, null, null, usageService));
     }
 
     public void testUnrecognizedMetric() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
index 38edbee673ee5..9098971e8e105 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
@@ -43,9 +43,9 @@ public class RestIndicesStatsActionTests extends ESTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        UsageService usageService = new UsageService(Settings.EMPTY);
+        UsageService usageService = new UsageService();
         action = new RestIndicesStatsAction(Settings.EMPTY,
-                new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null, usageService));
+                new RestController(Collections.emptySet(), null, null, null, usageService));
     }
 
     public void testUnrecognizedMetric() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index 2cb7bb9121a7c..59bdddb1fe3da 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -55,8 +55,8 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase {
     private static ThreadPool threadPool = new TestThreadPool(RestValidateQueryActionTests.class.getName());
     private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool);
 
-    private static UsageService usageService = new UsageService(Settings.EMPTY);
-    private static RestController controller = new RestController(Settings.EMPTY, emptySet(), null, client, null, usageService);
+    private static UsageService usageService = new UsageService();
+    private static RestController controller = new RestController(emptySet(), null, client, null, usageService);
     private static RestValidateQueryAction action = new RestValidateQueryAction(Settings.EMPTY, controller);
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 4535bf7a91b0d..674bdcd48462b 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -75,8 +75,8 @@ public class RestIndicesActionTests extends ESTestCase {
 
     public void testBuildTable() {
         final Settings settings = Settings.EMPTY;
-        UsageService usageService = new UsageService(settings);
-        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null, usageService);
+        UsageService usageService = new UsageService();
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService);
         final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings));
 
         // build a (semi-)random table
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestNodesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestNodesActionTests.java
index 32993a6b7c720..bf3d40af5e0e9 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestNodesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestNodesActionTests.java
@@ -47,9 +47,9 @@ public class RestNodesActionTests extends ESTestCase {
 
     @Before
     public void setUpAction() {
-        UsageService usageService = new UsageService(Settings.EMPTY);
+        UsageService usageService = new UsageService();
         action = new RestNodesAction(Settings.EMPTY,
-                new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null, usageService));
+                new RestController(Collections.emptySet(), null, null, null, usageService));
     }
 
     public void testBuildTableDoesNotThrowGivenNullNodeInfoAndStats() {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index e99fb4cc1f258..25f04532ac8ce 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -51,8 +51,8 @@ public class RestRecoveryActionTests extends ESTestCase {
 
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
-        UsageService usageService = new UsageService(settings);
-        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null, usageService);
+        UsageService usageService = new UsageService();
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService);
         final RestRecoveryAction action = new RestRecoveryAction(settings, restController);
         final int totalShards = randomIntBetween(1, 32);
         final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2));
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index fcb868c0e0e84..3c601ae9aaefc 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -54,6 +55,7 @@ public class ScriptServiceTests extends ESTestCase {
     private Map<String, ScriptContext<?>> contexts;
     private ScriptService scriptService;
     private Settings baseSettings;
+    private ClusterSettings clusterSettings;
 
     @Before
     public void setup() throws IOException {
@@ -78,12 +80,22 @@ public void setup() throws IOException {
     private void buildScriptService(Settings additionalSettings) throws IOException {
         Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build();
         scriptService = new ScriptService(finalSettings, engines, contexts) {
+            @Override
+            Map<String, StoredScriptSource> getScriptsFromClusterState() {
+                Map<String, StoredScriptSource> scripts = new HashMap<>();
+                scripts.put("test1", new StoredScriptSource("test", "1+1", Collections.emptyMap()));
+                scripts.put("test2", new StoredScriptSource("test", "1", Collections.emptyMap()));
+                return scripts;
+            }
+
             @Override
             StoredScriptSource getScriptFromClusterState(String id) {
                 //mock the script that gets retrieved from an index
                 return new StoredScriptSource("test", "1+1", Collections.emptyMap());
             }
         };
+        clusterSettings = new ClusterSettings(finalSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        scriptService.registerClusterSettingsListeners(clusterSettings);
     }
 
     // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes
@@ -305,6 +317,24 @@ public void testGetStoredScript() throws Exception {
         assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id")));
     }
 
+    public void testMaxSizeLimit() throws Exception {
+        buildScriptService(Settings.builder().put(ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.getKey(), 4).build());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values()));
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
+            scriptService.compile(new Script(ScriptType.INLINE, "test", "10+10", Collections.emptyMap()), randomFrom(contexts.values()));
+        });
+        assertEquals("exceeded max allowed inline script size in bytes [4] with size [5] for script [10+10]", iae.getMessage());
+        clusterSettings.applySettings(Settings.builder().put(ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.getKey(), 6).build());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "10+10", Collections.emptyMap()), randomFrom(contexts.values()));
+        clusterSettings.applySettings(Settings.builder().put(ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.getKey(), 5).build());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "10+10", Collections.emptyMap()), randomFrom(contexts.values()));
+        iae = expectThrows(IllegalArgumentException.class, () -> {
+            clusterSettings.applySettings(Settings.builder().put(ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.getKey(), 2).build());
+        });
+        assertEquals("script.max_size_in_bytes cannot be set to [2], stored script [test1] exceeds the new value with a size of [3]",
+                iae.getMessage());
+    }
+
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
             scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index ed9b8992577d8..37fc9cac029fe 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Strings;
@@ -85,6 +86,7 @@
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.DELETED;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -552,4 +554,17 @@ public void testSetSearchThrottled() {
         // we still make sure can match is executed on the network thread
         service.canMatch(req, ActionListener.wrap(r -> assertSame(Thread.currentThread(), currentThread), e -> fail("unexpected")));
     }
+
+    public void testExpandSearchThrottled() {
+        createIndex("throttled_threadpool_index");
+        client().execute(
+            InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.INSTANCE,
+            new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request("throttled_threadpool_index",
+                IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), "true"))
+            .actionGet();
+
+        client().prepareIndex("throttled_threadpool_index", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
+        assertHitCount(client().prepareSearch().get(), 0L);
+        assertHitCount(client().prepareSearch().setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_FORBID_CLOSED).get(), 1L);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
index e381240ca1f61..f10c3847a12c7 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
@@ -55,6 +55,7 @@
 import org.elasticsearch.search.aggregations.bucket.terms.StringTermsTests;
 import org.elasticsearch.search.aggregations.metrics.InternalExtendedStatsTests;
 import org.elasticsearch.search.aggregations.metrics.InternalMaxTests;
+import org.elasticsearch.search.aggregations.metrics.InternalMedianAbsoluteDeviationTests;
 import org.elasticsearch.search.aggregations.metrics.InternalMinTests;
 import org.elasticsearch.search.aggregations.metrics.InternalStatsBucketTests;
 import org.elasticsearch.search.aggregations.metrics.InternalStatsTests;
@@ -148,6 +149,7 @@ private static List<InternalAggregationTestCase> getAggsTests() {
         aggsTests.add(new InternalBinaryRangeTests());
         aggsTests.add(new InternalTopHitsTests());
         aggsTests.add(new InternalCompositeTests());
+        aggsTests.add(new InternalMedianAbsoluteDeviationTests());
         return Collections.unmodifiableList(aggsTests);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java
new file mode 100644
index 0000000000000..e15411343d136
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.search.aggregations.metrics.mad.InternalMedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.mad.ParsedMedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.InternalAggregationTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InternalMedianAbsoluteDeviationTests extends InternalAggregationTestCase<InternalMedianAbsoluteDeviation> {
+
+    @Override
+    protected InternalMedianAbsoluteDeviation createTestInstance(String name,
+                                                                 List<PipelineAggregator> pipelineAggregators,
+                                                                 Map<String, Object> metaData) {
+
+        final TDigestState valuesSketch = new TDigestState(randomDoubleBetween(20, 1000, true));
+        final int numberOfValues = frequently()
+            ? randomIntBetween(0, 1000)
+            : 0;
+        for (int i = 0; i < numberOfValues; i++) {
+            valuesSketch.add(randomDouble());
+        }
+
+        return new InternalMedianAbsoluteDeviation(name, pipelineAggregators, metaData, randomNumericDocValueFormat(), valuesSketch);
+    }
+
+    @Override
+    protected void assertReduced(InternalMedianAbsoluteDeviation reduced, List<InternalMedianAbsoluteDeviation> inputs) {
+        final TDigestState expectedValuesSketch = new TDigestState(reduced.getValuesSketch().compression());
+
+        long totalCount = 0;
+        for (InternalMedianAbsoluteDeviation input : inputs) {
+            expectedValuesSketch.add(input.getValuesSketch());
+            totalCount += input.getValuesSketch().size();
+        }
+
+        assertEquals(totalCount, reduced.getValuesSketch().size());
+        if (totalCount > 0) {
+            assertEquals(expectedValuesSketch.quantile(0), reduced.getValuesSketch().quantile(0), 0d);
+            assertEquals(expectedValuesSketch.quantile(1), reduced.getValuesSketch().quantile(1), 0d);
+        }
+    }
+
+    @Override
+    protected void assertFromXContent(InternalMedianAbsoluteDeviation internalMAD, ParsedAggregation parsedAggregation) throws IOException {
+        assertTrue(parsedAggregation instanceof ParsedMedianAbsoluteDeviation);
+        ParsedMedianAbsoluteDeviation parsedMAD = (ParsedMedianAbsoluteDeviation) parsedAggregation;
+        // Double.compare handles NaN, which we use for no result
+        assertEquals(internalMAD.getMedianAbsoluteDeviation(), parsedMAD.getMedianAbsoluteDeviation(), 0);
+    }
+
+    @Override
+    protected Writeable.Reader<InternalMedianAbsoluteDeviation> instanceReader() {
+        return InternalMedianAbsoluteDeviation::new;
+    }
+
+    @Override
+    protected InternalMedianAbsoluteDeviation mutateInstance(InternalMedianAbsoluteDeviation instance) throws IOException {
+        String name = instance.getName();
+        TDigestState valuesSketch = instance.getValuesSketch();
+        Map<String, Object> metaData = instance.getMetaData();
+
+        switch (between(0, 2)) {
+            case 0:
+                name += randomAlphaOfLengthBetween(2, 10);
+                break;
+            case 1:
+                final TDigestState newValuesSketch = new TDigestState(instance.getValuesSketch().compression());
+                final int numberOfValues = between(10, 100);
+                for (int i = 0; i < numberOfValues; i++) {
+                    newValuesSketch.add(randomDouble());
+                }
+                valuesSketch = newValuesSketch;
+                break;
+            case 2:
+                if (metaData == null) {
+                    metaData = new HashMap<>(1);
+                } else {
+                    metaData = new HashMap<>(metaData);
+                }
+                metaData.put(randomAlphaOfLengthBetween(2, 10), randomInt());
+                break;
+        }
+
+        return new InternalMedianAbsoluteDeviation(name, instance.pipelineAggregators(), metaData, instance.format, valuesSketch);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java
new file mode 100644
index 0000000000000..30dae4921b75c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorTests.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.metrics.mad.InternalMedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregator;
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+
+import static java.util.Collections.singleton;
+import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.ExactMedianAbsoluteDeviation.calculateMAD;
+import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.IsCloseToRelative.closeToRelative;
+import static org.hamcrest.Matchers.equalTo;
+
+public class MedianAbsoluteDeviationAggregatorTests extends AggregatorTestCase {
+
+    private static final int SAMPLE_MIN = -1000000;
+    private static final int SAMPLE_MAX = 1000000;
+
+    private static <T extends IndexableField> CheckedConsumer<RandomIndexWriter, IOException> randomSample(
+            int size,
+            Function<Long, Iterable<T>> field) {
+
+        return writer -> {
+            for (int i = 0; i < size; i++) {
+                final long point = randomLongBetween(SAMPLE_MIN, SAMPLE_MAX);
+                Iterable<T> document = field.apply(point);
+                writer.addDocument(document);
+            }
+        };
+    }
+
+    // intentionally not writing any docs
+    public void testNoDocs() throws IOException {
+        testCase(new MatchAllDocsQuery(), writer -> {}, agg -> assertThat(agg.getMedianAbsoluteDeviation(), equalTo(Double.NaN)));
+    }
+
+    public void testNoMatchingField() throws IOException {
+        testCase(
+            new MatchAllDocsQuery(),
+            writer -> {
+                writer.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
+                writer.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 2)));
+            },
+            agg -> assertThat(agg.getMedianAbsoluteDeviation(), equalTo(Double.NaN))
+        );
+    }
+
+    public void testSomeMatchesSortedNumericDocValues() throws IOException {
+        final int size = randomIntBetween(100, 1000);
+        final List<Long> sample = new ArrayList<>(size);
+        testCase(
+            new DocValuesFieldExistsQuery("number"),
+            randomSample(size, point -> {
+                sample.add(point);
+                return singleton(new SortedNumericDocValuesField("number", point));
+            }),
+            agg -> assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)))
+        );
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        final int size = randomIntBetween(100, 1000);
+        final List<Long> sample = new ArrayList<>(size);
+        testCase(
+            new DocValuesFieldExistsQuery("number"),
+            randomSample(size, point -> {
+                sample.add(point);
+                return singleton(new NumericDocValuesField("number", point));
+            }),
+            agg -> assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(sample)))
+        );
+    }
+
+    public void testQueryFiltering() throws IOException {
+        final int lowerRange = 1;
+        final int upperRange = 500;
+        final int[] sample = IntStream.rangeClosed(1, 1000).toArray();
+        final int[] filteredSample = Arrays.stream(sample).filter(point -> point  >= lowerRange && point <= upperRange).toArray();
+        testCase(
+            IntPoint.newRangeQuery("number", lowerRange, upperRange),
+            writer -> {
+                for (int point : sample) {
+                    writer.addDocument(Arrays.asList(new IntPoint("number", point), new SortedNumericDocValuesField("number", point)));
+                }
+            },
+            agg -> assertThat(agg.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(filteredSample)))
+        );
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        testCase(
+            IntPoint.newRangeQuery("number", -1, 0),
+            writer -> {
+                writer.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1)));
+                writer.addDocument(Arrays.asList(new IntPoint("number", 2), new SortedNumericDocValuesField("number", 2)));
+            },
+            agg -> assertThat(agg.getMedianAbsoluteDeviation(), equalTo(Double.NaN))
+        );
+    }
+
+    private void testCase(Query query,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalMedianAbsoluteDeviation> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                buildIndex.accept(indexWriter);
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad")
+                    .field("number")
+                    .compression(randomDoubleBetween(20, 1000, true));
+
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                fieldType.setName("number");
+
+                MedianAbsoluteDeviationAggregator aggregator = createAggregator(builder, indexSearcher, fieldType);
+                aggregator.preCollection();
+                indexSearcher.search(query, aggregator);
+                aggregator.postCollection();
+
+                verify.accept((InternalMedianAbsoluteDeviation) aggregator.buildAggregation(0L));
+            }
+        }
+
+    }
+
+    public static class IsCloseToRelative extends TypeSafeMatcher<Double> {
+
+        private final double expected;
+        private final double error;
+
+        public IsCloseToRelative(double expected, double error) {
+            this.expected = expected;
+            this.error = error;
+        }
+
+        @Override
+        protected boolean matchesSafely(Double actual) {
+            final double deviation = Math.abs(actual - expected);
+            final double observedError = deviation / Math.abs(expected);
+            return observedError <= error;
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            description
+                .appendText("within ")
+                .appendValue(error * 100)
+                .appendText(" percent of ")
+                .appendValue(expected);
+        }
+
+        public static IsCloseToRelative closeToRelative(double expected, double error) {
+            return new IsCloseToRelative(expected, error);
+        }
+
+        public static IsCloseToRelative closeToRelative(double expected) {
+            return closeToRelative(expected, 0.1);
+        }
+    }
+
+    /**
+     * This class is an implementation of median absolute deviation that computes an exact value, rather than the approximation used in the
+     * aggregation. It's used to verify that the aggregation's approximate results are close enough to the exact result
+     */
+    public static class ExactMedianAbsoluteDeviation {
+
+        public static double calculateMAD(int[] sample) {
+            return calculateMAD(Arrays.stream(sample)
+                .mapToDouble(point -> (double) point)
+                .toArray());
+        }
+
+        public static double calculateMAD(long[] sample) {
+            return calculateMAD(Arrays.stream(sample)
+                .mapToDouble(point -> (double) point)
+                .toArray());
+        }
+
+        public static double calculateMAD(List<Long> sample) {
+            return calculateMAD(sample.stream()
+                .mapToDouble(Long::doubleValue)
+                .toArray());
+        }
+
+        public static double calculateMAD(double[] sample) {
+            final double median = calculateMedian(sample);
+
+            final double[] deviations = Arrays.stream(sample)
+                .map(point -> Math.abs(median - point))
+                .toArray();
+
+            final double mad = calculateMedian(deviations);
+            return mad;
+        }
+
+        private static double calculateMedian(double[] sample) {
+            final double[] sorted = Arrays.copyOf(sample, sample.length);
+            Arrays.sort(sorted);
+
+            final int halfway =  (int) Math.ceil(sorted.length / 2d);
+            final double median = (sorted[halfway - 1] + sorted[halfway]) / 2d;
+            return median;
+        }
+
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
new file mode 100644
index 0000000000000..a4ef1785ceae7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationIT.java
@@ -0,0 +1,623 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.bucket.global.Global;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviation;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.LongStream;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.range;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.IsCloseToRelative.closeToRelative;
+import static org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregatorTests.ExactMedianAbsoluteDeviation.calculateMAD;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+public class MedianAbsoluteDeviationIT extends AbstractNumericTestCase {
+
+    private static final int MIN_SAMPLE_VALUE = -1000000;
+    private static final int MAX_SAMPLE_VALUE = 1000000;
+    private static final int NUMBER_OF_DOCS = 1000;
+    private static final Supplier<Long> sampleSupplier = () -> randomLongBetween(MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE);
+
+    private static long[] singleValueSample;
+    private static long[] multiValueSample;
+    private static double singleValueExactMAD;
+    private static double multiValueExactMAD;
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        final Settings settings = Settings.builder()
+            .put(SETTING_NUMBER_OF_SHARDS, 1)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+
+        createIndex("idx", settings);
+        createIndex("idx_unmapped", settings);
+
+        minValue = MIN_SAMPLE_VALUE;
+        minValues = MIN_SAMPLE_VALUE;
+        maxValue = MAX_SAMPLE_VALUE;
+        maxValues = MAX_SAMPLE_VALUE;
+
+        singleValueSample = new long[NUMBER_OF_DOCS];
+        multiValueSample = new long[NUMBER_OF_DOCS * 2];
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+
+        for (int i = 0; i < NUMBER_OF_DOCS; i++) {
+            final long singleValueDatapoint = sampleSupplier.get();
+            final long firstMultiValueDatapoint = sampleSupplier.get();
+            final long secondMultiValueDatapoint = sampleSupplier.get();
+
+            singleValueSample[i] = singleValueDatapoint;
+            multiValueSample[i * 2] = firstMultiValueDatapoint;
+            multiValueSample[(i * 2) + 1] = secondMultiValueDatapoint;
+
+            IndexRequestBuilder builder = client().prepareIndex("idx", "_doc", String.valueOf(i))
+                .setSource(jsonBuilder()
+                    .startObject()
+                        .field("value", singleValueDatapoint)
+                        .startArray("values")
+                            .value(firstMultiValueDatapoint)
+                            .value(secondMultiValueDatapoint)
+                        .endArray()
+                    .endObject());
+
+            builders.add(builder);
+        }
+
+        singleValueExactMAD = calculateMAD(singleValueSample);
+        multiValueExactMAD = calculateMAD(multiValueSample);
+
+        indexRandom(true, builders);
+
+        prepareCreate("empty_bucket_idx")
+            .addMapping("type", "value", "type=integer")
+            .execute()
+            .actionGet();
+
+        builders = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            builders.add(client().prepareIndex("empty_bucket_idx", "type", String.valueOf(i)).setSource(jsonBuilder()
+                .startObject()
+                .field("value", i*2)
+                .endObject()));
+        }
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(AggregationTestScriptsPlugin.class);
+    }
+
+    private static MedianAbsoluteDeviationAggregationBuilder randomBuilder() {
+        final MedianAbsoluteDeviationAggregationBuilder builder = new MedianAbsoluteDeviationAggregationBuilder("mad");
+        if (randomBoolean()) {
+            builder.compression(randomDoubleBetween(20, 1000, false));
+        }
+        return builder;
+    }
+
+    @Override
+    public void testEmptyAggregation() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("empty_bucket_idx")
+            .addAggregation(
+                histogram("histogram")
+                    .field("value")
+                    .interval(1)
+                    .minDocCount(0)
+                    .subAggregation(
+                        randomBuilder()
+                            .field("value")))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, 2);
+
+        final Histogram histogram = response.getAggregations().get("histogram");
+        assertThat(histogram, notNullValue());
+        final Histogram.Bucket bucket = histogram.getBuckets().get(1);
+        assertThat(bucket, notNullValue());
+
+        final MedianAbsoluteDeviation mad = bucket.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), is(Double.NaN));
+    }
+
+    @Override
+    public void testUnmapped() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx_unmapped")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("value"))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, 0);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), is(Double.NaN));
+    }
+
+    @Override
+    public void testSingleValuedField() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("value"))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(singleValueExactMAD));
+    }
+
+    @Override
+    public void testSingleValuedFieldGetProperty() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                global("global")
+                    .subAggregation(
+                        randomBuilder()
+                            .field("value")))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final Global global = response.getAggregations().get("global");
+        assertThat(global, notNullValue());
+        assertThat(global.getName(), is("global"));
+        assertThat(global.getDocCount(), is((long) NUMBER_OF_DOCS));
+        assertThat(global.getAggregations(), notNullValue());
+        assertThat(global.getAggregations().asMap().entrySet(), hasSize(1));
+
+        final MedianAbsoluteDeviation mad = global.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(((InternalAggregation) global).getProperty("mad"), sameInstance(mad));
+    }
+
+    @Override
+    public void testSingleValuedFieldPartiallyUnmapped() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx", "idx_unmapped")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("value"))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(singleValueExactMAD));
+    }
+
+    @Override
+    public void testSingleValuedFieldWithValueScript() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("value")
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", Collections.emptyMap())))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(singleValueSample)
+            .map(point -> point + 1)
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    @Override
+    public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
+        final Map<String, Object> params = new HashMap<>();
+        params.put("inc", 1);
+
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("value")
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(singleValueSample)
+            .map(point -> point + 1)
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    @Override
+    public void testMultiValuedField() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("values"))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(multiValueExactMAD));
+    }
+
+    @Override
+    public void testMultiValuedFieldWithValueScript() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("values")
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + 1", Collections.emptyMap())))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(multiValueSample)
+            .map(point -> point + 1)
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    @Override
+    public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
+        final Map<String, Object> params = new HashMap<>();
+        params.put("inc", 1);
+
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .field("values")
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value + inc", params)))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(multiValueSample)
+            .map(point -> point + 1)
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    @Override
+    public void testScriptSingleValued() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value", Collections.emptyMap())))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(singleValueExactMAD));
+    }
+
+    @Override
+    public void testScriptSingleValuedWithParams() throws Exception {
+        final Map<String, Object> params = new HashMap<>();
+        params.put("inc", 1);
+
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "doc['value'].value + inc", params)))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(singleValueSample)
+            .map(point -> point + 1)
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    @Override
+    public void testScriptMultiValued() throws Exception {
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .script(new Script(
+                        ScriptType.INLINE,
+                        AggregationTestScriptsPlugin.NAME,
+                        "doc['values'].values",
+                        Collections.emptyMap())))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(multiValueExactMAD));
+    }
+
+    @Override
+    public void testScriptMultiValuedWithParams() throws Exception {
+        final Map<String, Object> params = new HashMap<>();
+        params.put("inc", 1);
+
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                randomBuilder()
+                    .script(new Script(
+                        ScriptType.INLINE,
+                        AggregationTestScriptsPlugin.NAME,
+                        "[ doc['value'].value, doc['value'].value + inc ]",
+                        params)))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final MedianAbsoluteDeviation mad = response.getAggregations().get("mad");
+        assertThat(mad, notNullValue());
+        assertThat(mad.getName(), is("mad"));
+
+        final double fromIncrementedSampleMAD = calculateMAD(Arrays.stream(singleValueSample)
+            .flatMap(point -> LongStream.of(point, point + 1))
+            .toArray());
+        assertThat(mad.getMedianAbsoluteDeviation(), closeToRelative(fromIncrementedSampleMAD));
+    }
+
+    public void testAsSubAggregation() throws Exception {
+        final int rangeBoundary = (MAX_SAMPLE_VALUE + MIN_SAMPLE_VALUE) / 2;
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                range("range")
+                    .field("value")
+                    .addRange(MIN_SAMPLE_VALUE, rangeBoundary)
+                    .addRange(rangeBoundary, MAX_SAMPLE_VALUE)
+                    .subAggregation(
+                        randomBuilder()
+                            .field("value")))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final long[] lowerBucketSample = Arrays.stream(singleValueSample)
+            .filter(point -> point >= MIN_SAMPLE_VALUE && point < rangeBoundary)
+            .toArray();
+        final long[] upperBucketSample = Arrays.stream(singleValueSample)
+            .filter(point -> point >= rangeBoundary && point < MAX_SAMPLE_VALUE)
+            .toArray();
+
+        final Range range = response.getAggregations().get("range");
+        assertThat(range, notNullValue());
+        List<? extends Range.Bucket> buckets = range.getBuckets();
+        assertThat(buckets, notNullValue());
+        assertThat(buckets, hasSize(2));
+
+        final Range.Bucket lowerBucket = buckets.get(0);
+        assertThat(lowerBucket, notNullValue());
+
+        final MedianAbsoluteDeviation lowerBucketMAD = lowerBucket.getAggregations().get("mad");
+        assertThat(lowerBucketMAD, notNullValue());
+        assertThat(lowerBucketMAD.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(lowerBucketSample)));
+
+        final Range.Bucket upperBucket = buckets.get(1);
+        assertThat(upperBucket, notNullValue());
+
+        final MedianAbsoluteDeviation upperBucketMAD = upperBucket.getAggregations().get("mad");
+        assertThat(upperBucketMAD, notNullValue());
+        assertThat(upperBucketMAD.getMedianAbsoluteDeviation(), closeToRelative(calculateMAD(upperBucketSample)));
+
+    }
+
+    @Override
+    public void testOrderByEmptyAggregation() throws Exception {
+        final int numberOfBuckets = 10;
+        final SearchResponse response = client()
+            .prepareSearch("idx")
+            .setQuery(matchAllQuery())
+            .addAggregation(
+                terms("terms")
+                    .field("value")
+                    .size(numberOfBuckets)
+                    .order(BucketOrder.compound(BucketOrder.aggregation("filter>mad", true)))
+                    .subAggregation(
+                        filter("filter", termQuery("value", MAX_SAMPLE_VALUE + 1))
+                            .subAggregation(
+                                randomBuilder()
+                                    .field("value"))))
+            .execute()
+            .actionGet();
+
+        assertHitCount(response, NUMBER_OF_DOCS);
+
+        final Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        List<? extends Terms.Bucket> buckets = terms.getBuckets();
+        assertThat(buckets, notNullValue());
+        assertThat(buckets, hasSize(numberOfBuckets));
+
+        for (int i = 0; i < numberOfBuckets; i++) {
+            Terms.Bucket bucket = buckets.get(i);
+            assertThat(bucket, notNullValue());
+
+            Filter filter = bucket.getAggregations().get("filter");
+            assertThat(filter, notNullValue());
+            assertThat(filter.getDocCount(), equalTo(0L));
+
+            MedianAbsoluteDeviation mad = filter.getAggregations().get("mad");
+            assertThat(mad, notNullValue());
+            assertThat(mad.getMedianAbsoluteDeviation(), equalTo(Double.NaN));
+        }
+    }
+
+    /**
+     * Make sure that a request using a script does not get cached and a request
+     * not using a script does get cached.
+     */
+    public void testDontCacheScripts() throws Exception {
+        assertAcked(
+            prepareCreate("cache_test_idx")
+                .addMapping("type", "d", "type=long")
+                .setSettings(Settings.builder()
+                    .put("requests.cache.enable", true)
+                    .put("number_of_shards", 1)
+                    .put("number_of_replicas", 1))
+            .get());
+
+        indexRandom(true,
+            client().prepareIndex("cache_test_idx", "type", "1").setSource("s", 1),
+            client().prepareIndex("cache_test_idx", "type", "2").setSource("s", 2));
+
+        // Make sure we are starting with a clear cache
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getMissCount(), equalTo(0L));
+
+        // Test that a request using a script does not get cached
+        SearchResponse r = client().prepareSearch("cache_test_idx").setSize(0)
+            .addAggregation(randomBuilder()
+                .field("d")
+                .script(new Script(ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, "_value - 1", emptyMap()))).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getMissCount(), equalTo(0L));
+
+        // To make sure that the cache is working test that a request not using
+        // a script is cached
+        r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(randomBuilder().field("d")).get();
+        assertSearchResponse(r);
+
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getHitCount(), equalTo(0L));
+        assertThat(client().admin().indices().prepareStats("cache_test_idx").setRequestCache(true).get().getTotal().getRequestCache()
+            .getMissCount(), equalTo(1L));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationTests.java
new file mode 100644
index 0000000000000..6729624c324da
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationTests.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+
+public class MedianAbsoluteDeviationTests extends AbstractNumericMetricTestCase<MedianAbsoluteDeviationAggregationBuilder> {
+
+    @Override
+    protected MedianAbsoluteDeviationAggregationBuilder doCreateTestAggregatorFactory() {
+        MedianAbsoluteDeviationAggregationBuilder builder =
+            new MedianAbsoluteDeviationAggregationBuilder(randomAlphaOfLengthBetween(1, 20));
+
+        if (randomBoolean()) {
+            builder.compression(randomDoubleBetween(0, 1000.0, false));
+        }
+
+        if (randomBoolean()) {
+            builder.missing("MISSING");
+        }
+
+        if (randomBoolean()) {
+            builder.format("###.00");
+        }
+
+        return builder;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java
index b622c74314251..fcf7fc3979053 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricAggStateV6CompatTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptedMetricAggContexts;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
@@ -63,9 +62,9 @@ protected InternalScriptedMetric createTestInstance(String name, List<PipelineAg
      */
     @Override
     protected ScriptService mockScriptService() {
-        Function<Map<String, Object>, Object> script = params -> {
-            Object aggs = params.get("_aggs");
-            Object states = params.get("states");
+        Function<Map<String, Object>, Object> script = vars -> {
+            Object aggs = ((Map<String,Object>) vars.get("params")).get("_aggs");
+            Object states = vars.get("states");
             assertThat(aggs, instanceOf(List.class));
             assertThat(aggs, sameInstance(states));
             return aggs;
@@ -80,7 +79,8 @@ protected ScriptService mockScriptService() {
 
     @Override
     protected void assertReduced(InternalScriptedMetric reduced, List<InternalScriptedMetric> inputs) {
-        assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING);
+        assertWarnings("Accessing variable [_aggs] via [params._aggs] from within a scripted metric agg reduce script " +
+            "is deprecated in favor of using [state].");
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java
index 3e0413b3da562..fa541ffd149bf 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorAggStateV6CompatTests.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptedMetricAggContexts;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
@@ -75,40 +74,40 @@ public class ScriptedMetricAggregatorAggStateV6CompatTests extends AggregatorTes
     @SuppressWarnings("unchecked")
     public static void initMockScripts() {
         // If _agg is provided implicitly, it should be the same objects as "state" from the context.
-        SCRIPTS.put("initScript", params -> {
-            Object agg = params.get("_agg");
-            Object state = params.get("state");
+        SCRIPTS.put("initScript", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
+            Object state = vars.get("state");
             assertThat(agg, instanceOf(Map.class));
             assertThat(agg, sameInstance(state));
             return agg;
         });
-        SCRIPTS.put("mapScript", params -> {
-            Object agg = params.get("_agg");
-            Object state = params.get("state");
+        SCRIPTS.put("mapScript", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
+            Object state = vars.get("state");
             assertThat(agg, instanceOf(Map.class));
             assertThat(agg, sameInstance(state));
             return agg;
         });
-        SCRIPTS.put("combineScript", params -> {
-            Object agg = params.get("_agg");
-            Object state = params.get("state");
+        SCRIPTS.put("combineScript", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
+            Object state = vars.get("state");
             assertThat(agg, instanceOf(Map.class));
             assertThat(agg, sameInstance(state));
             return agg;
         });
 
-        SCRIPTS.put("initScriptExplicitAgg", params -> {
-            Object agg = params.get("_agg");
+        SCRIPTS.put("initScriptExplicitAgg", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
             assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT));
             return agg;
         });
-        SCRIPTS.put("mapScriptExplicitAgg", params -> {
-            Object agg = params.get("_agg");
+        SCRIPTS.put("mapScriptExplicitAgg", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
             assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT));
             return agg;
         });
-        SCRIPTS.put("combineScriptExplicitAgg", params -> {
-            Object agg = params.get("_agg");
+        SCRIPTS.put("combineScriptExplicitAgg", vars -> {
+            Object agg = ((Map<String,Object>) vars.get("params")).get("_agg");
             assertThat(agg, equalTo(EXPLICIT_AGG_OBJECT));
             return agg;
         });
@@ -132,7 +131,12 @@ public void testWithImplicitAggParam() throws IOException {
             }
         }
 
-        assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING,
+        assertWarnings("Accessing variable [_agg] via [params._agg] from within a scripted metric agg init script " +
+                "is deprecated in favor of using [state].",
+            "Accessing variable [_agg] via [params._agg] from within a scripted metric agg map script " +
+                "is deprecated in favor of using [state].",
+            "Accessing variable [_agg] via [params._agg] from within a scripted metric agg combine script " +
+                "is deprecated in favor of using [state].",
             "[reduceScript] must be provided for metric aggregations.");
     }
 
@@ -162,7 +166,12 @@ public void testWithExplicitAggParam() throws IOException {
             }
         }
 
-        assertWarnings(ScriptedMetricAggContexts.AGG_PARAM_DEPRECATION_WARNING,
+        assertWarnings("Accessing variable [_agg] via [params._agg] from within a scripted metric agg init script " +
+                "is deprecated in favor of using [state].",
+            "Accessing variable [_agg] via [params._agg] from within a scripted metric agg map script " +
+                "is deprecated in favor of using [state].",
+            "Accessing variable [_agg] via [params._agg] from within a scripted metric agg combine script " +
+                "is deprecated in favor of using [state].",
             "[reduceScript] must be provided for metric aggregations.");
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
index 034d6a129db14..565c3a775141c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java
@@ -107,7 +107,7 @@ public static void initMockScripts() {
 
         SCRIPTS.put("initScriptParams", params -> {
             Map<String, Object> state = (Map<String, Object>) params.get("state");
-            Integer initialValue = (Integer)params.get("initialValue");
+            Integer initialValue = (Integer)((Map<String,Object>)params.get("params")).get("initialValue");
             ArrayList<Integer> collector = new ArrayList();
             collector.add(initialValue);
             state.put("collector", collector);
@@ -115,13 +115,13 @@ public static void initMockScripts() {
         });
         SCRIPTS.put("mapScriptParams", params -> {
             Map<String, Object> state = (Map<String, Object>) params.get("state");
-            Integer itemValue = (Integer) params.get("itemValue");
+            Integer itemValue = (Integer) ((Map<String,Object>)params.get("params")).get("itemValue");
             ((List<Integer>) state.get("collector")).add(itemValue);
             return state;
         });
         SCRIPTS.put("combineScriptParams", params -> {
             Map<String, Object> state = (Map<String, Object>) params.get("state");
-            int divisor = ((Integer) params.get("divisor"));
+            int divisor = ((Integer) ((Map<String,Object>)params.get("params")).get("divisor"));
             return ((List<Integer>) state.get("collector")).stream().mapToInt(Integer::intValue).map(i -> i / divisor).sum();
         });
     }
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
index 5523bcca51263..02a775983f68d 100644
--- a/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
@@ -67,7 +67,7 @@ public void setUp() throws Exception {
         when(mapperService.fullName("alias")).thenReturn(fieldType);
 
         FieldInfo mockFieldInfo = new FieldInfo("field", 1, false, false, true,
-            IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, false);
+            IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
 
         LeafReader leafReader = mock(LeafReader.class);
         doAnswer(invocation -> {
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 77f7aed844a66..d043e106ced4f 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -18,9 +18,6 @@
  */
 package org.elasticsearch.transport;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Supplier;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
@@ -75,7 +72,9 @@
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -86,6 +85,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -170,7 +170,7 @@ public void testRemoteProfileIsUsedForLocalCluster() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -211,7 +211,7 @@ public void testRemoteProfileIsUsedForRemoteCluster() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -263,7 +263,7 @@ public void testDiscoverSingleNode() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -292,7 +292,7 @@ public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, seedNodes);
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -319,7 +319,7 @@ public void testNodeDisconnected() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -369,7 +369,7 @@ public void testFilterDiscoveredNodes() throws Exception {
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
                     Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE,
-                    n -> n.equals(rejectedNode) == false)) {
+                    n -> n.equals(rejectedNode) == false, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     if (rejectedNode.equals(seedNode)) {
                         assertFalse(service.nodeConnected(seedNode));
@@ -412,7 +412,7 @@ public void testConnectWithIncompatibleTransports() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     expectThrows(Exception.class, () -> updateSeedNodes(connection, Arrays.asList(() -> seedNode)));
                     assertFalse(service.nodeConnected(seedNode));
                     assertTrue(connection.assertNoRunningConnections());
@@ -475,7 +475,7 @@ public void close() {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     connection.addConnectedNode(seedNode);
                     for (DiscoveryNode node : knownNodes) {
                         final Transport.Connection transportConnection = connection.getConnection(node);
@@ -518,7 +518,7 @@ public void run() {
                 CountDownLatch listenerCalled = new CountDownLatch(1);
                 AtomicReference<Exception> exceptionReference = new AtomicReference<>();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     ActionListener<Void> listener = ActionListener.wrap(x -> {
                         listenerCalled.countDown();
                         fail("expected exception");
@@ -555,7 +555,7 @@ public void testFetchShards() throws Exception {
                 service.acceptIncomingRequests();
                 List<Supplier<DiscoveryNode>> nodes = Collections.singletonList(() -> seedNode);
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    nodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    nodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     if (randomBoolean()) {
                         updateSeedNodes(connection, nodes);
                     }
@@ -595,7 +595,7 @@ public void testFetchShardsThreadContextHeader() throws Exception {
                 service.acceptIncomingRequests();
                 List<Supplier<DiscoveryNode>> nodes = Collections.singletonList(() -> seedNode);
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    nodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    nodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     SearchRequest request = new SearchRequest("test-index");
                     Thread[] threads = new Thread[10];
                     for (int i = 0; i < threads.length; i++) {
@@ -649,7 +649,8 @@ public void testFetchShardsSkipUnavailable() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Collections.singletonList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Collections.singletonList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE,
+                    n -> true, null)) {
 
                     SearchRequest request = new SearchRequest("test-index");
                     ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest("test-index")
@@ -661,7 +662,7 @@ public void testFetchShardsSkipUnavailable() throws Exception {
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
                             new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
-                        assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
+                        assertTrue(responseLatch.await(10, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
                         ClusterSearchShardsResponse response = reference.get();
@@ -692,7 +693,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
                             new LatchedActionListener<>(ActionListener.wrap((s) -> {
                                 reference.set(s);
                             }, failReference::set), responseLatch));
-                        assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
+                        assertTrue(responseLatch.await(10, TimeUnit.SECONDS));
                         assertNotNull(failReference.get());
                         assertNull(reference.get());
                         assertThat(failReference.get(), instanceOf(TransportException.class));
@@ -705,7 +706,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
                             new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
-                        assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
+                        assertTrue(responseLatch.await(10, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
                         ClusterSearchShardsResponse response = reference.get();
@@ -714,7 +715,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
 
                     //give transport service enough time to realize that the node is down, and to notify the connection listeners
                     //so that RemoteClusterConnection is left with no connected nodes, hence it will retry connecting next
-                    assertTrue(disconnectedLatch.await(1, TimeUnit.SECONDS));
+                    assertTrue(disconnectedLatch.await(10, TimeUnit.SECONDS));
 
                     if (randomBoolean()) {
                         connection.updateSkipUnavailable(false);
@@ -728,7 +729,7 @@ public void onNodeDisconnected(DiscoveryNode node) {
                         AtomicReference<Exception> failReference = new AtomicReference<>();
                         connection.fetchSearchShards(searchShardsRequest,
                             new LatchedActionListener<>(ActionListener.wrap(reference::set, failReference::set), responseLatch));
-                        assertTrue(responseLatch.await(1, TimeUnit.SECONDS));
+                        assertTrue(responseLatch.await(10, TimeUnit.SECONDS));
                         assertNull(failReference.get());
                         assertNotNull(reference.get());
                         ClusterSearchShardsResponse response = reference.get();
@@ -759,7 +760,7 @@ public void testTriggerUpdatesConcurrently() throws IOException, InterruptedExce
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     int numThreads = randomIntBetween(4, 10);
                     Thread[] threads = new Thread[numThreads];
                     CyclicBarrier barrier = new CyclicBarrier(numThreads);
@@ -838,7 +839,7 @@ public void testCloseWhileConcurrentlyConnecting() throws IOException, Interrupt
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     int numThreads = randomIntBetween(4, 10);
                     Thread[] threads = new Thread[numThreads];
                     CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
@@ -943,7 +944,7 @@ public void testGetConnectionInfo() throws Exception {
                 service.acceptIncomingRequests();
                 int maxNumConnections = randomIntBetween(1, 5);
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    seedNodes, service, service.connectionManager(), maxNumConnections, n -> true)) {
+                    seedNodes, service, service.connectionManager(), maxNumConnections, n -> true, null)) {
                     // test no nodes connected
                     RemoteConnectionInfo remoteConnectionInfo = assertSerialization(getRemoteConnectionInfo(connection));
                     assertNotNull(remoteConnectionInfo);
@@ -1138,7 +1139,7 @@ public void testEnsureConnected() throws IOException, InterruptedException {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     assertFalse(service.nodeConnected(seedNode));
                     assertFalse(service.nodeConnected(discoverableNode));
                     assertTrue(connection.assertNoRunningConnections());
@@ -1187,7 +1188,7 @@ public void testCollectNodes() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(() -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     if (randomBoolean()) {
                         updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     }
@@ -1235,7 +1236,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    seedNodes, service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     final int numGetThreads = randomIntBetween(4, 10);
                     final Thread[] getThreads = new Thread[numGetThreads];
                     final int numModifyingThreads = randomIntBetween(4, 10);
@@ -1325,7 +1326,7 @@ public void testClusterNameIsChecked() throws Exception {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList( () -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList( () -> seedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(() -> seedNode));
                     assertTrue(service.nodeConnected(seedNode));
                     assertTrue(service.nodeConnected(discoverableNode));
@@ -1405,7 +1406,8 @@ public void close() {
                 service.start();
                 service.acceptIncomingRequests();
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                Collections.singletonList(() -> connectedNode), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Collections.singletonList(() -> connectedNode), service, service.getConnectionManager(),
+                    Integer.MAX_VALUE, n -> true, null)) {
                     connection.addConnectedNode(connectedNode);
                     for (int i = 0; i < 10; i++) {
                         //always a direct connection as the remote node is already connected
@@ -1447,7 +1449,7 @@ public void testLazyResolveTransportAddress() throws Exception {
                     return seedNode;
                 };
                 try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
-                    Arrays.asList(seedSupplier), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true)) {
+                    Arrays.asList(seedSupplier), service, service.getConnectionManager(), Integer.MAX_VALUE, n -> true, null)) {
                     updateSeedNodes(connection, Arrays.asList(seedSupplier));
                     // Closing connections leads to RemoteClusterConnection.ConnectHandler.collectRemoteNodes
                     // being called again so we try to resolve the same seed node's host twice
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
index d24f921219b8a..c7840e36b1144 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -96,6 +97,7 @@ public void testSettingsAreRegistered() {
         assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER));
         assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING));
         assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_NODE_ATTRIBUTE));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_CLUSTER_PING_SCHEDULE));
     }
 
     public void testRemoteClusterSeedSetting() {
@@ -193,12 +195,12 @@ public void testBuildRemoteClustersDynamicConfigWithDuplicates() {
 
     public void testGroupClusterIndices() throws IOException {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
-        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
-             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+        try (MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
+            knownNodes.add(cluster1Transport.getLocalDiscoNode());
+            knownNodes.add(cluster2Transport.getLocalDiscoNode());
             Collections.shuffle(knownNodes, random());
 
             try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
@@ -206,8 +208,8 @@ public void testGroupClusterIndices() throws IOException {
                 transportService.start();
                 transportService.acceptIncomingRequests();
                 Settings.Builder builder = Settings.builder();
-                builder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString());
-                builder.putList("cluster.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                builder.putList("cluster.remote.cluster_1.seeds", cluster1Seed.getAddress().toString());
+                builder.putList("cluster.remote.cluster_2.seeds", cluster2Seed.getAddress().toString());
                 try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
                     assertFalse(service.isCrossClusterSearchEnabled());
                     service.initializeRemoteClusters();
@@ -238,12 +240,12 @@ public void testGroupClusterIndices() throws IOException {
 
     public void testGroupIndices() throws IOException {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
-        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
-             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+        try (MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
+            knownNodes.add(cluster1Transport.getLocalDiscoNode());
+            knownNodes.add(cluster2Transport.getLocalDiscoNode());
             Collections.shuffle(knownNodes, random());
 
             try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
@@ -251,8 +253,8 @@ public void testGroupIndices() throws IOException {
                 transportService.start();
                 transportService.acceptIncomingRequests();
                 Settings.Builder builder = Settings.builder();
-                builder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString());
-                builder.putList("cluster.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                builder.putList("cluster.remote.cluster_1.seeds", cluster1Seed.getAddress().toString());
+                builder.putList("cluster.remote.cluster_2.seeds", cluster2Seed.getAddress().toString());
                 try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
                     assertFalse(service.isCrossClusterSearchEnabled());
                     service.initializeRemoteClusters();
@@ -300,12 +302,12 @@ public void testGroupIndices() throws IOException {
 
     public void testIncrementallyAddClusters() throws IOException {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
-        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
-             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
-            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
-            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
-            knownNodes.add(seedTransport.getLocalDiscoNode());
-            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+        try (MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
+            knownNodes.add(cluster1Transport.getLocalDiscoNode());
+            knownNodes.add(cluster2Transport.getLocalDiscoNode());
             Collections.shuffle(knownNodes, random());
 
             try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
@@ -313,16 +315,16 @@ public void testIncrementallyAddClusters() throws IOException {
                 transportService.start();
                 transportService.acceptIncomingRequests();
                 Settings.Builder builder = Settings.builder();
-                builder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString());
-                builder.putList("cluster.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                builder.putList("cluster.remote.cluster_1.seeds", cluster1Seed.getAddress().toString());
+                builder.putList("cluster.remote.cluster_2.seeds", cluster2Seed.getAddress().toString());
                 try (RemoteClusterService service = new RemoteClusterService(Settings.EMPTY, transportService)) {
                     assertFalse(service.isCrossClusterSearchEnabled());
                     service.initializeRemoteClusters();
                     assertFalse(service.isCrossClusterSearchEnabled());
-                    service.updateRemoteCluster("cluster_1", Collections.singletonList(seedNode.getAddress().toString()), null);
+                    service.updateRemoteCluster("cluster_1", Collections.singletonList(cluster1Seed.getAddress().toString()), null);
                     assertTrue(service.isCrossClusterSearchEnabled());
                     assertTrue(service.isRemoteClusterRegistered("cluster_1"));
-                    service.updateRemoteCluster("cluster_2", Collections.singletonList(otherSeedNode.getAddress().toString()), null);
+                    service.updateRemoteCluster("cluster_2", Collections.singletonList(cluster2Seed.getAddress().toString()), null);
                     assertTrue(service.isCrossClusterSearchEnabled());
                     assertTrue(service.isRemoteClusterRegistered("cluster_1"));
                     assertTrue(service.isRemoteClusterRegistered("cluster_2"));
@@ -336,6 +338,81 @@ public void testIncrementallyAddClusters() throws IOException {
         }
     }
 
+    public void testDefaultPingSchedule() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            TimeValue pingSchedule;
+            Settings.Builder settingsBuilder = Settings.builder();
+            settingsBuilder.putList("cluster.remote.cluster_1.seeds", seedNode.getAddress().toString());
+            if (randomBoolean()) {
+                pingSchedule = TimeValue.timeValueSeconds(randomIntBetween(1, 10));
+                settingsBuilder.put(TcpTransport.PING_SCHEDULE.getKey(), pingSchedule).build();
+            } else {
+                pingSchedule = TimeValue.MINUS_ONE;
+            }
+            Settings settings = settingsBuilder.build();
+            try (MockTransportService transportService = MockTransportService.createNewService(settings,
+                Version.CURRENT, threadPool, null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                try (RemoteClusterService service = new RemoteClusterService(settings, transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    service.updateRemoteCluster("cluster_1", Collections.singletonList(seedNode.getAddress().toString()), null);
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    RemoteClusterConnection remoteClusterConnection = service.getRemoteClusterConnection("cluster_1");
+                    assertEquals(pingSchedule, remoteClusterConnection.getConnectionManager().getPingSchedule());
+                }
+            }
+        }
+    }
+
+    public void testCustomPingSchedule() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode cluster1Seed = cluster1Transport.getLocalDiscoNode();
+            DiscoveryNode cluster2Seed = cluster2Transport.getLocalDiscoNode();
+            knownNodes.add(cluster1Transport.getLocalDiscoNode());
+            knownNodes.add(cluster2Transport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            Settings.Builder settingsBuilder = Settings.builder();
+            if (randomBoolean()) {
+                settingsBuilder.put(TcpTransport.PING_SCHEDULE.getKey(), TimeValue.timeValueSeconds(randomIntBetween(1, 10)));
+            }
+            Settings transportSettings = settingsBuilder.build();
+
+            try (MockTransportService transportService = MockTransportService.createNewService(transportSettings, Version.CURRENT,
+                threadPool, null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putList("cluster.remote.cluster_1.seeds", cluster1Seed.getAddress().toString());
+                builder.putList("cluster.remote.cluster_2.seeds", cluster2Seed.getAddress().toString());
+                TimeValue pingSchedule1 = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueSeconds(randomIntBetween(1, 10));
+                builder.put("cluster.remote.cluster_1.transport.ping_schedule", pingSchedule1);
+                TimeValue pingSchedule2 = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueSeconds(randomIntBetween(1, 10));
+                builder.put("cluster.remote.cluster_2.transport.ping_schedule", pingSchedule2);
+                try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    service.updateRemoteCluster("cluster_1", Collections.singletonList(cluster1Seed.getAddress().toString()), null);
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    RemoteClusterConnection remoteClusterConnection1 = service.getRemoteClusterConnection("cluster_1");
+                    assertEquals(pingSchedule1, remoteClusterConnection1.getConnectionManager().getPingSchedule());
+                    RemoteClusterConnection remoteClusterConnection2 = service.getRemoteClusterConnection("cluster_2");
+                    assertEquals(pingSchedule2, remoteClusterConnection2.getConnectionManager().getPingSchedule());
+                }
+            }
+        }
+    }
+
     public void testRemoteNodeAttribute() throws IOException, InterruptedException {
         final Settings settings =
                 Settings.builder().put("cluster.remote.node.attr", "gateway").build();
diff --git a/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java b/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java
index c6347b014bfb8..c38030afbe973 100644
--- a/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java
@@ -52,7 +52,7 @@ public void testRestUsage() throws Exception {
         BaseRestHandler handlerD = new MockRestHandler("d", settings);
         BaseRestHandler handlerE = new MockRestHandler("e", settings);
         BaseRestHandler handlerF = new MockRestHandler("f", settings);
-        UsageService usageService = new UsageService(settings);
+        UsageService usageService = new UsageService();
         usageService.addRestHandler(handlerA);
         usageService.addRestHandler(handlerB);
         usageService.addRestHandler(handlerC);
diff --git a/settings.gradle b/settings.gradle
index fae2b6efc7d34..3be6d9152a033 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -24,10 +24,10 @@ List projects = [
   'distribution:packages:deb',
   'distribution:packages:oss-rpm',
   'distribution:packages:rpm',
-  'distribution:bwc:next-minor-snapshot',
-  'distribution:bwc:staged-minor-snapshot',
-  'distribution:bwc:next-bugfix-snapshot',
-  'distribution:bwc:maintenance-bugfix-snapshot',
+  'distribution:bwc:bugfix',
+  'distribution:bwc:maintenance',
+  'distribution:bwc:minor',
+  'distribution:bwc:staged',
   'distribution:tools:java-version-checker',
   'distribution:tools:launchers',
   'distribution:tools:plugin-cli',
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 1c95991413214..ccdd2876b4c16 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -72,4 +72,5 @@ precommit.dependsOn namingConventionsMain
 test.configure {
   systemProperty 'tests.gradle_index_compat_versions', bwcVersions.indexCompatible.join(',')
   systemProperty 'tests.gradle_wire_compat_versions', bwcVersions.wireCompatible.join(',')
+  systemProperty 'tests.gradle_unreleased_versions', bwcVersions.unreleased.join(',')
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index e1205ba846b36..b3681247b4e9c 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -70,19 +70,19 @@ public static MockAllocationService createAllocationService(Settings settings, R
     }
 
     public static MockAllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings, Random random) {
-        return new MockAllocationService(settings,
+        return new MockAllocationService(
                 randomAllocationDeciders(settings, clusterSettings, random),
                 new TestGatewayAllocator(), new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
-        return new MockAllocationService(settings,
+        return new MockAllocationService(
                 randomAllocationDeciders(settings, EMPTY_CLUSTER_SETTINGS, random()),
             new TestGatewayAllocator(), new BalancedShardsAllocator(settings), clusterInfoService);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator gatewayAllocator) {
-        return new MockAllocationService(settings,
+        return new MockAllocationService(
                 randomAllocationDeciders(settings, EMPTY_CLUSTER_SETTINGS, random()),
                 gatewayAllocator, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
     }
@@ -91,7 +91,7 @@ public static AllocationDeciders randomAllocationDeciders(Settings settings, Clu
         List<AllocationDecider> deciders = new ArrayList<>(
             ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList()));
         Collections.shuffle(deciders, random);
-        return new AllocationDeciders(settings, deciders);
+        return new AllocationDeciders(deciders);
     }
 
     protected static Set<DiscoveryNode.Role> MASTER_DATA_ROLES =
@@ -127,18 +127,18 @@ protected  static ClusterState startRandomInitializingShard(ClusterState cluster
     }
 
     protected static AllocationDeciders yesAllocationDeciders() {
-        return new AllocationDeciders(Settings.EMPTY, Arrays.asList(
+        return new AllocationDeciders(Arrays.asList(
             new TestAllocateDecision(Decision.YES),
             new SameShardAllocationDecider(Settings.EMPTY,
                                            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
     }
 
     protected static AllocationDeciders noAllocationDeciders() {
-        return new AllocationDeciders(Settings.EMPTY, Collections.singleton(new TestAllocateDecision(Decision.NO)));
+        return new AllocationDeciders(Collections.singleton(new TestAllocateDecision(Decision.NO)));
     }
 
     protected static AllocationDeciders throttleAllocationDeciders() {
-        return new AllocationDeciders(Settings.EMPTY, Arrays.asList(
+        return new AllocationDeciders(Arrays.asList(
             new TestAllocateDecision(Decision.THROTTLE),
             new SameShardAllocationDecider(Settings.EMPTY,
                                            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
@@ -159,7 +159,6 @@ public static class TestAllocateDecision extends AllocationDecider {
         private final Decision decision;
 
         public TestAllocateDecision(Decision decision) {
-            super(Settings.EMPTY);
             this.decision = decision;
         }
 
@@ -184,9 +183,9 @@ protected static class MockAllocationService extends AllocationService {
 
         private volatile long nanoTimeOverride = -1L;
 
-        public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
+        public MockAllocationService(AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
                                      ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
-            super(settings, allocationDeciders, gatewayAllocator, shardsAllocator, clusterInfoService);
+            super(allocationDeciders, gatewayAllocator, shardsAllocator, clusterInfoService);
         }
 
         public void setNanoTimeOverride(long nanoTime) {
@@ -203,10 +202,7 @@ protected long currentNanoTime() {
      * Mocks behavior in ReplicaShardAllocator to remove delayed shards from list of unassigned shards so they don't get reassigned yet.
      */
     protected static class DelayedShardsMockGatewayAllocator extends GatewayAllocator {
-
-        public DelayedShardsMockGatewayAllocator() {
-            super(Settings.EMPTY);
-        }
+        public DelayedShardsMockGatewayAllocator() {}
 
         @Override
         public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java
index d4d2d78789b8e..e8a554ca4aaae 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/alias/RandomAliasActionsGenerator.java
@@ -82,6 +82,9 @@ public static AliasActions randomAliasAction(boolean useStringAsFilter) {
                     action.indexRouting(randomRouting().toString());
                 }
             }
+            if (randomBoolean()) {
+                action.writeIndex(randomBoolean());
+            }
         }
         return action;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index 7ed23f2ea3a63..972d631e2f700 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -573,7 +573,14 @@ public EngineConfig config(IndexSettings indexSettings, Store store, Path transl
 
     public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy,
                                ReferenceManager.RefreshListener refreshListener, Sort indexSort, LongSupplier globalCheckpointSupplier) {
-        IndexWriterConfig iwc = newIndexWriterConfig();
+        return config(indexSettings, store, translogPath, mergePolicy, refreshListener, null, indexSort, globalCheckpointSupplier);
+    }
+
+    public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy,
+                               ReferenceManager.RefreshListener externalRefreshListener,
+                               ReferenceManager.RefreshListener internalRefreshListener,
+                               Sort indexSort, LongSupplier globalCheckpointSupplier) {
+            IndexWriterConfig iwc = newIndexWriterConfig();
         TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
         Engine.EventListener listener = new Engine.EventListener() {
             @Override
@@ -581,12 +588,14 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
-        final List<ReferenceManager.RefreshListener> refreshListenerList =
-                refreshListener == null ? emptyList() : Collections.singletonList(refreshListener);
+        final List<ReferenceManager.RefreshListener> extRefreshListenerList =
+            externalRefreshListener == null ? emptyList() : Collections.singletonList(externalRefreshListener);
+        final List<ReferenceManager.RefreshListener> intRefreshListenerList =
+            internalRefreshListener == null ? emptyList() : Collections.singletonList(internalRefreshListener);
         EngineConfig config = new EngineConfig(shardId, allocationId.getId(), threadPool, indexSettings, null, store,
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
-                TimeValue.timeValueMinutes(5), refreshListenerList, Collections.emptyList(), indexSort,
+                TimeValue.timeValueMinutes(5), extRefreshListenerList, intRefreshListenerList, indexSort,
                 new NoneCircuitBreakerService(),
                 globalCheckpointSupplier == null ?
                     new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED, update -> {}) :
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 4e9f491ea04c1..6234dcf72ecda 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -167,7 +167,7 @@ protected class ReplicationGroup implements AutoCloseable, Iterable<IndexShard>
         boolean closed = false;
         private ReplicationTargets replicationTargets;
 
-        private final PrimaryReplicaSyncer primaryReplicaSyncer = new PrimaryReplicaSyncer(Settings.EMPTY,
+        private final PrimaryReplicaSyncer primaryReplicaSyncer = new PrimaryReplicaSyncer(
             new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()),
             (request, parentTask, primaryAllocationId, primaryTerm, listener) -> {
                 try {
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index c813b42319c99..1562705f6f463 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -422,7 +422,7 @@ public void execute() {
             Map<String, Object> map = new HashMap<>();
 
             if (getParams() != null) {
-                map.putAll(getParams()); // TODO: remove this once scripts know to look for params under params key
+                map.putAll(getParams());
                 map.put("params", getParams());
             }
 
@@ -453,7 +453,7 @@ public void execute() {
                     Map<String, Object> map = new HashMap<>();
 
                     if (getParams() != null) {
-                        map.putAll(getParams()); // TODO: remove this once scripts know to look for params under params key
+                        map.putAll(getParams());
                         map.put("params", getParams());
                     }
 
@@ -480,7 +480,7 @@ public Object execute() {
             Map<String, Object> map = new HashMap<>();
 
             if (getParams() != null) {
-                map.putAll(getParams()); // TODO: remove this once scripts know to look for params under params key
+                map.putAll(getParams());
                 map.put("params", getParams());
             }
 
@@ -502,7 +502,7 @@ public Object execute() {
             Map<String, Object> map = new HashMap<>();
 
             if (getParams() != null) {
-                map.putAll(getParams()); // TODO: remove this once scripts know to look for params under params key
+                map.putAll(getParams());
                 map.put("params", getParams());
             }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 7fa3d934f1b27..3393ed51fec50 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -412,7 +412,6 @@ protected boolean builderGeneratesCacheableQueries() {
      * Test creates the {@link Query} from the {@link QueryBuilder} under test and delegates the
      * assertions being made on the result to the implementing subclass.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34188")
     public void testToQuery() throws IOException {
         for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) {
             QueryShardContext context = createShardContext();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index efc553354e9e7..021be808d22a2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -131,8 +131,11 @@ public static ClusterService createClusterService(ThreadPool threadPool, Discove
     }
 
     public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode, ClusterSettings clusterSettings) {
-        ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
-            clusterSettings, threadPool, Collections.emptyMap());
+        Settings settings = Settings.builder()
+                .put("node.name", "test")
+                .put("cluster.name", "ClusterServiceTests")
+                .build();
+        ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool, Collections.emptyMap());
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
             public void connectToNodes(DiscoveryNodes discoveryNodes) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index dc59440875ef6..823ae5cc49919 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -69,6 +69,7 @@
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
@@ -183,9 +184,9 @@
 @LuceneTestCase.SuppressReproduceLine
 public abstract class ESTestCase extends LuceneTestCase {
 
-    private static final List<String> JODA_TIMEZONE_IDS;
-    private static final List<String> JAVA_TIMEZONE_IDS;
-    private static final List<String> JAVA_ZONE_IDS;
+    protected static final List<String> JODA_TIMEZONE_IDS;
+    protected static final List<String> JAVA_TIMEZONE_IDS;
+    protected static final List<String> JAVA_ZONE_IDS;
 
     private static final AtomicInteger portGenerator = new AtomicInteger();
 
@@ -226,8 +227,9 @@ public void append(LogEvent event) {
 
         BootstrapForTesting.ensureInitialized();
 
-        List<String> jodaTZIds = new ArrayList<>(DateTimeZone.getAvailableIDs());
-        Collections.sort(jodaTZIds);
+        // filter out joda timezones that are deprecated for the java time migration
+        List<String> jodaTZIds = DateTimeZone.getAvailableIDs().stream()
+            .filter(s -> DateUtils.DEPRECATED_SHORT_TZ_IDS.contains(s) == false).sorted().collect(Collectors.toList());
         JODA_TIMEZONE_IDS = Collections.unmodifiableList(jodaTZIds);
 
         List<String> javaTZIds = Arrays.asList(TimeZone.getAvailableIDs());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index ada9d54b1b861..bed00526698c7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -117,6 +117,8 @@
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ParsedValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.MedianAbsoluteDeviationAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.mad.ParsedMedianAbsoluteDeviation;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -178,6 +180,7 @@ public abstract class InternalAggregationTestCase<T extends InternalAggregation>
         map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
         map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
         map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
+        map.put(MedianAbsoluteDeviationAggregationBuilder.NAME, (p, c) -> ParsedMedianAbsoluteDeviation.fromXContent(p, (String) c));
         map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
         map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
         map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 721f0249b2670..bfdcf718dcd6b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1336,12 +1336,7 @@ public void assertSeqNos() throws Exception {
                         } catch (AlreadyClosedException e) {
                             continue; // shard is closed - just ignore
                         }
-                        assertThat(replicaShardRouting + " local checkpoint mismatch",
-                            seqNoStats.getLocalCheckpoint(), equalTo(primarySeqNoStats.getLocalCheckpoint()));
-                        assertThat(replicaShardRouting + " global checkpoint mismatch",
-                            seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
-                        assertThat(replicaShardRouting + " max seq no mismatch",
-                            seqNoStats.getMaxSeqNo(), equalTo(primarySeqNoStats.getMaxSeqNo()));
+                        assertThat(replicaShardRouting + " seq_no_stats mismatch", seqNoStats, equalTo(primarySeqNoStats));
                         // the local knowledge on the primary of the global checkpoint equals the global checkpoint on the shard
                         assertThat(replicaShardRouting + " global checkpoint syncs mismatch", seqNoStats.getGlobalCheckpoint(),
                             equalTo(syncGlobalCheckpoints.get(replicaShardRouting.allocationId().getId())));
@@ -1590,6 +1585,7 @@ private synchronized void startAndPublishNodesAndClients(List<NodeAndClient> nod
                 // if we're adding too many master-eligible nodes at once, we can't update the min master setting before adding the nodes.
                 updateMinMasterNodes(currentMasters + newMasters);
             }
+            rebuildUnicastHostFiles(nodeAndClients); // ensure that new nodes can find the existing nodes when they start
             List<Future<?>> futures = nodeAndClients.stream().map(node -> executor.submit(node::startNode)).collect(Collectors.toList());
 
             try {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
index ae62acff9b734..37db06a15e6a9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.discovery.zen.PingContextProvider;
@@ -45,8 +44,7 @@ public final class MockZenPing extends AbstractComponent implements ZenPing {
 
     private final PingContextProvider contextProvider;
 
-    public MockZenPing(Settings settings, PingContextProvider contextProvider) {
-        super(settings);
+    public MockZenPing(PingContextProvider contextProvider) {
         this.contextProvider = contextProvider;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
index 2c8305b4e12bb..a42ee370ece43 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
@@ -99,7 +99,7 @@ private TestZenDiscovery(Settings settings, ThreadPool threadPool, TransportServ
     protected ZenPing newZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
                                  UnicastHostsProvider hostsProvider) {
         if (USE_MOCK_PINGS.get(settings)) {
-            return new MockZenPing(settings, this);
+            return new MockZenPing(this);
         } else {
             return super.newZenPing(settings, threadPool, transportService, hostsProvider);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java b/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
index d3e05d36f6ea5..9966bfb47fa47 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/gateway/NoopGatewayAllocator.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;
 
 import java.util.List;
@@ -34,10 +33,6 @@ public class NoopGatewayAllocator extends GatewayAllocator {
 
     public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
 
-    protected NoopGatewayAllocator() {
-        super(Settings.EMPTY);
-    }
-
     @Override
     public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
         // noop
diff --git a/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java
index 2bbf2ce4c2caf..bcbe52e32cd43 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/gateway/TestGatewayAllocator.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.AsyncShardFetch;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.gateway.PrimaryShardAllocator;
@@ -60,7 +59,7 @@ public class TestGatewayAllocator extends GatewayAllocator {
     Map<String /* node id */, Map<ShardId, ShardRouting>> knownAllocations = new HashMap<>();
     DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES;
 
-    PrimaryShardAllocator primaryShardAllocator = new PrimaryShardAllocator(Settings.EMPTY) {
+    PrimaryShardAllocator primaryShardAllocator = new PrimaryShardAllocator() {
         @Override
         protected AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
             // for now always return immediately what we know
@@ -81,7 +80,7 @@ protected AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardR
         }
     };
 
-    ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator(Settings.EMPTY) {
+    ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator() {
         @Override
         protected AsyncShardFetch.FetchResult<NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation) {
             // for now, just pretend no node has data
@@ -95,10 +94,6 @@ protected boolean hasInitiatedFetching(ShardRouting shard) {
         }
     };
 
-    public TestGatewayAllocator() {
-        super(Settings.EMPTY);
-    }
-
     @Override
     public void applyStartedShards(RoutingAllocation allocation, List<ShardRouting> startedShards) {
         currentNodes = allocation.nodes();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
index 5f64bcc21df97..56f77bcefc81a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.test.junit.listeners;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.junit.runner.Description;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index db0ee04e6239e..608b65ebb9e30 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -282,7 +282,8 @@ public static void waitForPendingTasks(final RestClient adminClient, final Predi
     /**
      * Returns whether to preserve the state of the cluster upon completion of this test. Defaults to false. If true, overrides the value of
      * {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()},
-     * {@link #preserveSnapshotsUponCompletion()}, and {@link #preserveRollupJobsUponCompletion()}.
+     * {@link #preserveSnapshotsUponCompletion()},{@link #preserveRollupJobsUponCompletion()},
+     * and {@link #preserveILMPoliciesUponCompletion()}.
      *
      * @return true if the state of the cluster should be preserved
      */
@@ -347,6 +348,15 @@ protected boolean preserveRollupJobsUponCompletion() {
         return false;
     }
 
+    /**
+     * Returns whether to preserve ILM Policies of this test. Defaults to not
+     * preserviing them. Only runs at all if xpack is installed on the cluster
+     * being tested.
+     */
+    protected boolean preserveILMPoliciesUponCompletion() {
+        return false;
+    }
+
     private void wipeCluster() throws Exception {
         if (preserveIndicesUponCompletion() == false) {
             // wipe indices
@@ -404,6 +414,10 @@ private void wipeCluster() throws Exception {
             wipeRollupJobs();
             waitForPendingRollupTasks();
         }
+
+        if (hasXPack && false == preserveILMPoliciesUponCompletion()) {
+            deleteAllPolicies();
+        }
     }
 
     /**
@@ -513,6 +527,26 @@ private void waitForPendingRollupTasks() throws Exception {
         waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("xpack/rollup/job") == false);
     }
 
+    private static void deleteAllPolicies() throws IOException {
+        Map<String, Object> policies;
+
+        try {
+            Response response = adminClient().performRequest(new Request("GET", "/_ilm/policy"));
+            policies = entityAsMap(response);
+        } catch (IOException e) {
+            // If bad request returned, ILM is not enabled.
+            return;
+        }
+
+        if (policies == null || policies.isEmpty()) {
+            return;
+        }
+
+        for (String policyName : policies.keySet()) {
+            adminClient().performRequest(new Request("DELETE", "/_ilm/policy/" + policyName));
+        }
+    }
+
     /**
      * Logs a message if there are still running tasks. The reasoning is that any tasks still running are state the is trying to bleed into
      * other tests.
@@ -698,6 +732,14 @@ protected static void createIndex(String name, Settings settings, String mapping
         client().performRequest(request);
     }
 
+    protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException {
+        Request request = new Request("PUT", "/" + name);
+        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
+            + ", \"mappings\" : {" + mapping + "}"
+            + ", \"aliases\": {" + aliases + "} }");
+        client().performRequest(request);
+    }
+
     protected static void deleteIndex(String name) throws IOException {
         Request request = new Request("DELETE", "/" + name);
         client().performRequest(request);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index bd75bb793a27b..739d5cb71ef3e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -39,7 +39,6 @@
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -190,19 +189,44 @@ public static Iterable<Object[]> createParameters() throws Exception {
      */
     public static Iterable<Object[]> createParameters(NamedXContentRegistry executeableSectionRegistry) throws Exception {
         String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
-        List<Object[]> tests = new ArrayList<>();
         Map<String, Set<Path>> yamlSuites = loadSuites(paths);
+        List<ClientYamlTestSuite> suites = new ArrayList<>();
+        IllegalArgumentException validationException = null;
         // yaml suites are grouped by directory (effectively by api)
         for (String api : yamlSuites.keySet()) {
             List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
             for (Path yamlFile : yamlFiles) {
-                ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile);
-                for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
-                    tests.add(new Object[]{ new ClientYamlTestCandidate(restTestSuite, testSection) });
+                ClientYamlTestSuite suite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile);
+                suites.add(suite);
+                try {
+                    suite.validate();
+                } catch(IllegalArgumentException e) {
+                    if (validationException == null) {
+                        validationException = new IllegalArgumentException("Validation errors for the following test suites:\n- "
+                            + e.getMessage());
+                    } else {
+                        String previousMessage = validationException.getMessage();
+                        Throwable[] suppressed = validationException.getSuppressed();
+                        validationException = new IllegalArgumentException(previousMessage + "\n- " + e.getMessage());
+                        for (Throwable t : suppressed) {
+                            validationException.addSuppressed(t);
+                        }
+                    }
+                    validationException.addSuppressed(e);
                 }
             }
         }
 
+        if (validationException != null) {
+            throw validationException;
+        }
+
+        List<Object[]> tests = new ArrayList<>();
+        for (ClientYamlTestSuite yamlTestSuite : suites) {
+            for (ClientYamlTestSection testSection : yamlTestSuite.getTestSections()) {
+                tests.add(new Object[]{ new ClientYamlTestCandidate(yamlTestSuite, testSection) });
+            }
+        }
         //sort the candidates so they will always be in the same order before being shuffled, for repeatability
         tests.sort(Comparator.comparing(o -> ((ClientYamlTestCandidate) o[0]).getTestPath()));
         return tests;
@@ -360,7 +384,7 @@ public void test() throws IOException {
             }
         } finally {
             logger.debug("start teardown test [{}]", testCandidate.getTestPath());
-            for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) {
+            for (ExecutableSection doSection : testCandidate.getTeardownSection().getDoSections()) {
                 executeSection(doSection);
             }
             logger.debug("end teardown test [{}]", testCandidate.getTestPath());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
index 1ec2382fac596..48e7fc031139b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
@@ -18,14 +18,15 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
-import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Represents a test section, which is composed of a skip section and multiple executable sections.
@@ -33,34 +34,37 @@
 public class ClientYamlTestSection implements Comparable<ClientYamlTestSection> {
     public static ClientYamlTestSection parse(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
-        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName());
+        XContentLocation sectionLocation = parser.getTokenLocation();
+        String sectionName = parser.currentName();
+        List<ExecutableSection> executableSections = new ArrayList<>();
         try {
             parser.nextToken();
-            testSection.setSkipSection(SkipSection.parseIfNext(parser));
+            SkipSection skipSection = SkipSection.parseIfNext(parser);
             while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
                 ParserUtils.advanceToFieldName(parser);
-                testSection.addExecutableSection(ExecutableSection.parse(parser));
+                executableSections.add(ExecutableSection.parse(parser));
             }
             if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                throw new IllegalArgumentException("malformed section [" + testSection.getName() + "] expected ["
+                throw new IllegalArgumentException("malformed section [" + sectionName + "] expected ["
                         + XContentParser.Token.END_OBJECT + "] but was [" + parser.currentToken() + "]");
             }
             parser.nextToken();
-            return testSection;
+            return new ClientYamlTestSection(sectionLocation, sectionName, skipSection, executableSections);
         } catch (Exception e) {
-            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e);
+            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + sectionName + "]", e);
         }
     }
 
     private final XContentLocation location;
     private final String name;
-    private SkipSection skipSection;
+    private final SkipSection skipSection;
     private final List<ExecutableSection> executableSections;
 
-    public ClientYamlTestSection(XContentLocation location, String name) {
+    ClientYamlTestSection(XContentLocation location, String name, SkipSection skipSection, List<ExecutableSection> executableSections) {
         this.location = location;
         this.name = name;
-        this.executableSections = new ArrayList<>();
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.executableSections = Collections.unmodifiableList(executableSections);
     }
 
     public XContentLocation getLocation() {
@@ -75,33 +79,10 @@ public SkipSection getSkipSection() {
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
     public List<ExecutableSection> getExecutableSections() {
         return executableSections;
     }
 
-    public void addExecutableSection(ExecutableSection executableSection) {
-        if (executableSection instanceof DoSection) {
-            DoSection doSection = (DoSection) executableSection;
-            if (false == doSection.getExpectedWarningHeaders().isEmpty()
-                    && false == skipSection.getFeatures().contains("warnings")) {
-                throw new IllegalArgumentException("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so "
-                        + "runners that do not support the [warnings] section can skip the test at line ["
-                        + doSection.getLocation().lineNumber + "]");
-            }
-            if (NodeSelector.ANY != doSection.getApiCallSection().getNodeSelector()
-                    && false == skipSection.getFeatures().contains("node_selector")) {
-                throw new IllegalArgumentException("Attempted to add a [do] with a [node_selector] section without a corresponding "
-                    + "[skip] so runners that do not support the [node_selector] section can skip the test at line ["
-                    + doSection.getLocation().lineNumber + "]");
-            }
-        }
-        this.executableSections.add(executableSection);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
index a8cc96461292a..61eac4fa03465 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -31,9 +32,13 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Holds a REST test suite loaded from a specific yaml file.
@@ -77,11 +82,9 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa
         assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
                 + parser.currentToken();
 
-        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName);
-
-        restTestSuite.setSetupSection(SetupSection.parseIfNext(parser));
-        restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser));
-
+        SetupSection setupSection = SetupSection.parseIfNext(parser);
+        TeardownSection teardownSection = TeardownSection.parseIfNext(parser);
+        Set<ClientYamlTestSection> testSections = new TreeSet<>();
         while(true) {
             //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
             //we need to somehow distinguish between a null in the middle of a test ("---")
@@ -91,27 +94,28 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa
                     break;
                 }
             }
-
             ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
-            if (!restTestSuite.addTestSection(testSection)) {
+            if (testSections.add(testSection) == false) {
                 throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]");
             }
         }
 
-        return restTestSuite;
+        return new ClientYamlTestSuite(api, suiteName, setupSection, teardownSection, new ArrayList<>(testSections));
     }
 
     private final String api;
     private final String name;
+    private final SetupSection setupSection;
+    private final TeardownSection teardownSection;
+    private final List<ClientYamlTestSection> testSections;
 
-    private SetupSection setupSection;
-    private TeardownSection teardownSection;
-
-    private Set<ClientYamlTestSection> testSections = new TreeSet<>();
-
-    public ClientYamlTestSuite(String api, String name) {
+    ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection,
+                        List<ClientYamlTestSection> testSections) {
         this.api = api;
         this.name = name;
+        this.setupSection = Objects.requireNonNull(setupSection, "setup section cannot be null");
+        this.teardownSection = Objects.requireNonNull(teardownSection, "teardown section cannot be null");
+        this.testSections = Collections.unmodifiableList(testSections);
     }
 
     public String getApi() {
@@ -130,27 +134,71 @@ public SetupSection getSetupSection() {
         return setupSection;
     }
 
-    public void setSetupSection(SetupSection setupSection) {
-        this.setupSection = setupSection;
-    }
-
     public TeardownSection getTeardownSection() {
         return teardownSection;
     }
 
-    public void setTeardownSection(TeardownSection teardownSection) {
-        this.teardownSection = teardownSection;
+    public void validate() {
+        Stream<String> errors = validateExecutableSections(setupSection.getExecutableSections(), null, setupSection, null);
+        errors = Stream.concat(errors, validateExecutableSections(teardownSection.getDoSections(), null, null, teardownSection));
+        errors = Stream.concat(errors, testSections.stream()
+            .flatMap(section -> validateExecutableSections(section.getExecutableSections(), section, setupSection, teardownSection)));
+        String errorMessage = errors.collect(Collectors.joining(",\n"));
+        if (errorMessage.isEmpty() == false) {
+            throw new IllegalArgumentException(getPath() + ":\n" + errorMessage);
+        }
+    }
+
+    private static Stream<String> validateExecutableSections(List<ExecutableSection> sections,
+                                                   ClientYamlTestSection testSection,
+                                                   SetupSection setupSection, TeardownSection teardownSection) {
+
+        Stream<String> errors = sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> false == section.getExpectedWarningHeaders().isEmpty())
+            .filter(section -> false == hasSkipFeature("warnings", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [warnings] section " +
+                "without a corresponding [\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] " +
+                "section can skip the test at line [" + section.getLocation().lineNumber + "]");
+
+        errors = Stream.concat(errors, sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> NodeSelector.ANY != section.getApiCallSection().getNodeSelector())
+            .filter(section -> false == hasSkipFeature("node_selector", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [node_selector] " +
+                "section without a corresponding [\"skip\": \"features\": \"node_selector\"] so runners that do not support the " +
+                "[node_selector] section can skip the test at line [" + section.getLocation().lineNumber + "]"));
+
+        errors = Stream.concat(errors, sections.stream()
+            .filter(section -> section instanceof ContainsAssertion)
+            .filter(section -> false == hasSkipFeature("contains", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [contains] assertion " +
+                "without a corresponding [\"skip\": \"features\": \"contains\"] so runners that do not support the " +
+                "[contains] assertion can skip the test at line [" + section.getLocation().lineNumber + "]"));
+
+        errors = Stream.concat(errors, sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> false == section.getApiCallSection().getHeaders().isEmpty())
+            .filter(section -> false == hasSkipFeature("headers", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [headers] section without a corresponding "
+                + "[\"skip\": \"features\": \"headers\"] so runners that do not support the [headers] section can skip the test at " +
+                "line [" + section.getLocation().lineNumber + "]"));
+
+        return errors;
+    }
+
+    private static boolean hasSkipFeature(String feature, ClientYamlTestSection testSection,
+                                          SetupSection setupSection, TeardownSection teardownSection) {
+        return (testSection != null && hasSkipFeature(feature, testSection.getSkipSection())) ||
+            (setupSection != null && hasSkipFeature(feature, setupSection.getSkipSection())) ||
+            (teardownSection != null && hasSkipFeature(feature, teardownSection.getSkipSection()));
     }
 
-    /**
-     * Adds a {@link org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection} to the REST suite
-     * @return true if the test section was not already present, false otherwise
-     */
-    public boolean addTestSection(ClientYamlTestSection testSection) {
-        return this.testSections.add(testSection);
+    private static boolean hasSkipFeature(String feature, SkipSection skipSection) {
+        return skipSection != null && skipSection.getFeatures().contains(feature);
     }
 
     public List<ClientYamlTestSection> getTestSections() {
-        return new ArrayList<>(testSections);
+        return testSections;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ContainsAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ContainsAssertion.java
new file mode 100644
index 0000000000000..2f7e74750a247
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ContainsAssertion.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class ContainsAssertion extends Assertion {
+    public static ContainsAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String, Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        return new ContainsAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
+
+    private static final Logger logger = Loggers.getLogger(ContainsAssertion.class);
+
+    public ContainsAssertion(XContentLocation location, String field, Object expectedValue) {
+        super(location, field, expectedValue);
+    }
+
+    @Override
+    protected void doAssert(Object actualValue, Object expectedValue) {
+        // add support for matching objects ({a:b}) against list of objects ([ {a:b, c:d} ])
+        if (expectedValue instanceof Map && actualValue instanceof List) {
+            logger.trace("assert that [{}] contains [{}]", actualValue, expectedValue);
+            Map<String, Object> expectedMap = (Map<String, Object>) expectedValue;
+            List<Object> actualList = (List<Object>) actualValue;
+            List<Map<String, Object>> actualValues = actualList.stream()
+                .filter(each -> each instanceof Map)
+                .map((each -> (Map<String, Object>) each))
+                .filter(each -> each.keySet().containsAll(expectedMap.keySet()))
+                .collect(Collectors.toList());
+            assertThat(
+                getField() + " expected to be a list with at least one object that has keys: " +
+                    expectedMap.keySet() + " but it was " + actualList,
+                actualValues,
+                is(not(empty()))
+            );
+            assertTrue(
+                getField() + " expected to be a list with at least on object that matches " + expectedMap +
+                    " but was " + actualValues,
+                actualValues.stream()
+                    .anyMatch(each -> each.entrySet().containsAll(expectedMap.entrySet()))
+            );
+        } else {
+            fail("'contains' only supports checking an object against a list of objects");
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index e9c5f58f3489b..05c723ad90586 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -106,6 +106,8 @@ public static DoSection parse(XContentParser parser) throws IOException {
             } else if (token.isValue()) {
                 if ("catch".equals(currentFieldName)) {
                     doSection.setCatch(parser.text());
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unsupported field [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if ("warnings".equals(currentFieldName)) {
@@ -180,7 +182,6 @@ public static DoSection parse(XContentParser parser) throws IOException {
         return doSection;
     }
 
-
     private static final Logger logger = LogManager.getLogger(DoSection.class);
 
     private final XContentLocation location;
@@ -204,7 +205,7 @@ public ApiCallSection getApiCallSection() {
         return apiCallSection;
     }
 
-    public void setApiCallSection(ApiCallSection apiCallSection) {
+    void setApiCallSection(ApiCallSection apiCallSection) {
         this.apiCallSection = apiCallSection;
     }
 
@@ -212,7 +213,7 @@ public void setApiCallSection(ApiCallSection apiCallSection) {
      * Warning headers that we expect from this response. If the headers don't match exactly this request is considered to have failed.
      * Defaults to emptyList.
      */
-    public List<String> getExpectedWarningHeaders() {
+    List<String> getExpectedWarningHeaders() {
         return expectedWarningHeaders;
     }
 
@@ -220,7 +221,7 @@ public List<String> getExpectedWarningHeaders() {
      * Set the warning headers that we expect from this response. If the headers don't match exactly this request is considered to have
      * failed. Defaults to emptyList.
      */
-    public void setExpectedWarningHeaders(List<String> expectedWarningHeaders) {
+    void setExpectedWarningHeaders(List<String> expectedWarningHeaders) {
         this.expectedWarningHeaders = expectedWarningHeaders;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
index ce5ea1c1cde06..ff02d6d16aa4a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
@@ -47,6 +47,7 @@ public interface ExecutableSection {
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("contains"), ContainsAssertion::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse)));
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
index 692888a003857..38a034d47e769 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
@@ -22,7 +22,9 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Represents a setup section. Holds a skip section and multiple do sections.
@@ -31,7 +33,7 @@ public class SetupSection {
     /**
      * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}.
      */
-    public static SetupSection parseIfNext(XContentParser parser) throws IOException {
+    static SetupSection parseIfNext(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
 
         if ("setup".equals(parser.currentName())) {
@@ -45,58 +47,42 @@ public static SetupSection parseIfNext(XContentParser parser) throws IOException
     }
 
     public static SetupSection parse(XContentParser parser) throws IOException {
-        SetupSection setupSection = new SetupSection();
-        setupSection.setSkipSection(SkipSection.parseIfNext(parser));
-
+        SkipSection skipSection = SkipSection.parseIfNext(parser);
+        List<ExecutableSection> executableSections = new ArrayList<>();
         while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
             ParserUtils.advanceToFieldName(parser);
             if ("do".equals(parser.currentName())) {
-                setupSection.addDoSection(DoSection.parse(parser));
+                executableSections.add(DoSection.parse(parser));
             } else if ("set".equals(parser.currentName())) {
-                setupSection.addSetSection(SetSection.parse(parser));
+                executableSections.add(SetSection.parse(parser));
             } else {
                 throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section");
             }
 
             parser.nextToken();
         }
-
         parser.nextToken();
-
-        return setupSection;
+        return new SetupSection(skipSection, executableSections);
     }
 
-    public static final SetupSection EMPTY;
+    public static final SetupSection EMPTY = new SetupSection(SkipSection.EMPTY, Collections.emptyList());
 
-    static {
-        EMPTY = new SetupSection();
-        EMPTY.setSkipSection(SkipSection.EMPTY);
-    }
-
-    private SkipSection skipSection;
+    private final SkipSection skipSection;
+    private final List<ExecutableSection> executableSections;
 
-    private List<ExecutableSection> executableSections = new ArrayList<>();
+    SetupSection(SkipSection skipSection, List<ExecutableSection> executableSections) {
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.executableSections = Collections.unmodifiableList(executableSections);
+    }
 
     public SkipSection getSkipSection() {
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
     public List<ExecutableSection> getExecutableSections() {
         return executableSections;
     }
 
-    public void addDoSection(DoSection doSection) {
-        this.executableSections.add(doSection);
-    }
-
-    public void addSetSection(SetSection setSection) {
-        this.executableSections.add(setSection);
-    }
-
     public boolean isEmpty() {
         return EMPTY.equals(this);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
index 3b272fe673c76..5eecaa758f0bc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
@@ -24,13 +24,15 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 public class TeardownSection {
     /**
      * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}.
      */
-    public static TeardownSection parseIfNext(XContentParser parser) throws IOException {
+    static TeardownSection parseIfNext(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
 
         if ("teardown".equals(parser.currentName())) {
@@ -44,50 +46,40 @@ public static TeardownSection parseIfNext(XContentParser parser) throws IOExcept
     }
 
     public static TeardownSection parse(XContentParser parser) throws IOException {
-        TeardownSection teardownSection = new TeardownSection();
-        teardownSection.setSkipSection(SkipSection.parseIfNext(parser));
-
+        SkipSection skipSection = SkipSection.parseIfNext(parser);
+        List<ExecutableSection> executableSections = new ArrayList<>();
         while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
             ParserUtils.advanceToFieldName(parser);
             if (!"do".equals(parser.currentName())) {
                 throw new ParsingException(parser.getTokenLocation(),
                         "section [" + parser.currentName() + "] not supported within teardown section");
             }
-
-            teardownSection.addDoSection(DoSection.parse(parser));
+            executableSections.add(DoSection.parse(parser));
             parser.nextToken();
         }
 
         parser.nextToken();
-        return teardownSection;
+        return new TeardownSection(skipSection, executableSections);
     }
 
-    public static final TeardownSection EMPTY;
+    public static final TeardownSection EMPTY = new TeardownSection(SkipSection.EMPTY, Collections.emptyList());
 
-    static {
-        EMPTY = new TeardownSection();
-        EMPTY.setSkipSection(SkipSection.EMPTY);
-    }
+    private final SkipSection skipSection;
+    private final List<ExecutableSection> doSections;
 
-    private SkipSection skipSection;
-    private List<DoSection> doSections = new ArrayList<>();
+    TeardownSection(SkipSection skipSection, List<ExecutableSection> doSections) {
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.doSections = Collections.unmodifiableList(doSections);
+    }
 
     public SkipSection getSkipSection() {
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
-    public List<DoSection> getDoSections() {
+    public List<ExecutableSection> getDoSections() {
         return doSections;
     }
 
-    public void addDoSection(DoSection doSection) {
-        this.doSections.add(doSection);
-    }
-
     public boolean isEmpty() {
         return EMPTY.equals(this);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java
index 861d574b3465e..84f9b4dfd149d 100644
--- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java
@@ -66,4 +66,9 @@ protected boolean preserveClusterSettings() {
     protected boolean preserveRollupJobsUponCompletion() {
         return true;
     }
+
+    @Override
+    protected boolean preserveILMPoliciesUponCompletion() {
+        return true;
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
index 240120a7a8530..f9b1a571c3a3c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
@@ -24,7 +24,9 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.equalTo;
@@ -368,13 +370,19 @@ private class VersionsFromProperty {
         private final List<String> unreleased = new ArrayList<>();
 
         private VersionsFromProperty(String property) {
+            Set<String> allUnreleased = new HashSet<>(Arrays.asList(
+                System.getProperty("tests.gradle_unreleased_versions", "").split(",")
+            ));
+            if (allUnreleased.isEmpty()) {
+                fail("[tests.gradle_unreleased_versions] not set or empty. Gradle should set this before running.");
+            }
             String versions = System.getProperty(property);
-            assertNotNull("Couldn't find [" + property + "]. Gradle should set these before running the tests.", versions);
+            assertNotNull("Couldn't find [" + property + "]. Gradle should set this before running the tests.", versions);
             logger.info("Looked up versions [{}={}]", property, versions);
 
             for (String version : versions.split(",")) {
-                if (version.endsWith("-SNAPSHOT")) {
-                    unreleased.add(version.replace("-SNAPSHOT", ""));
+                if (allUnreleased.contains(version)) {
+                    unreleased.add(version);
                 } else {
                     released.add(version);
                 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
index fcef74678359e..ddf153ff44f5c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
@@ -134,6 +134,22 @@ public void testParseMatchArray() throws Exception {
         assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testParseContains() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "{testKey: { someKey: someValue } }"
+        );
+
+        ContainsAssertion containsAssertion = ContainsAssertion.parse(parser);
+        assertThat(containsAssertion, notNullValue());
+        assertThat(containsAssertion.getField(), equalTo("testKey"));
+        assertThat(containsAssertion.getExpectedValue(), instanceOf(Map.class));
+        assertThat(
+            ((Map<String, String>) containsAssertion.getExpectedValue()).get("someKey"),
+            equalTo("someValue")
+        );
+    }
+
     @SuppressWarnings("unchecked")
     public void testParseMatchSourceValues() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
index 5da8601a9f340..94bc69fe714b3 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
@@ -20,78 +20,19 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 
 import java.io.IOException;
 import java.util.Map;
 
-import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testAddingDoWithoutSkips() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(SkipSection.EMPTY);
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithWarningWithSkip() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("warnings"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setExpectedWarningHeaders(singletonList("foo"));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithWarningWithSkipButNotWarnings() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setExpectedWarningHeaders(singletonList("foo"));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection));
-        assertEquals("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so runners that do not support the"
-                + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage());
-    }
-
-    public void testAddingDoWithNodeSelectorWithSkip() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("node_selector"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        ApiCallSection apiCall = new ApiCallSection("test");
-        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
-        doSection.setApiCallSection(apiCall);
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithNodeSelectorWithSkipButNotWarnings() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        ApiCallSection apiCall = new ApiCallSection("test");
-        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
-        doSection.setApiCallSection(apiCall);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection));
-        assertEquals("Attempted to add a [do] with a [node_selector] section without a corresponding"
-                + " [skip] so runners that do not support the [node_selector] section can skip the test at"
-                + " line [" + lineNumber + "]", e.getMessage());
-    }
-
     public void testWrongIndentation() throws Exception {
         {
             XContentParser parser = createParser(YamlXContent.yamlXContent,
@@ -297,7 +238,7 @@ public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
         LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
         assertThat(lengthAssertion.getField(), equalTo("_index"));
         assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
+        assertThat(lengthAssertion.getExpectedValue(), equalTo(6));
 
         IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
         assertThat(falseAssertion.getField(), equalTo("whatever"));
@@ -305,12 +246,12 @@ public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
         GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
         assertThat(greaterThanAssertion.getField(), equalTo("size"));
         assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
+        assertThat(greaterThanAssertion.getExpectedValue(), equalTo(5));
 
         LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
         assertThat(lessThanAssertion.getField(), equalTo("size"));
         assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
+        assertThat(lessThanAssertion.getExpectedValue(), equalTo(10));
     }
 
     public void testSmallSection() throws Exception {
@@ -327,5 +268,4 @@ public void testSmallSection() throws Exception {
         assertThat(testSection.getName(), equalTo("node_info test"));
         assertThat(testSection.getExecutableSections().size(), equalTo(3));
     }
-
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
index c97175da7ca53..1e831bdd6464e 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@@ -20,11 +20,18 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -106,9 +113,11 @@ public void testParseTestSetupTeardownAndSections() throws Exception {
             assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
             assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
             assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getApi(),
+                equalTo("indices.delete"));
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().size(),
+                equalTo(1));
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().get("index"),
                 equalTo("test_index"));
         } else {
             assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
@@ -378,4 +387,188 @@ public void testParseTestDuplicateTestSections() throws Exception {
             ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser));
         assertThat(e.getMessage(), containsString("duplicate test section"));
     }
+
+    public void testAddingDoWithoutSkips() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+            SkipSection.EMPTY, Collections.singletonList(doSection));
+        ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY,
+            Collections.singletonList(section));
+        clientYamlTestSuite.validate();
+    }
+
+    public void testAddingDoWithWarningWithoutSkipWarnings() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [warnings] section without a corresponding " +
+            "[\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] section can skip the test " +
+            "at line [" + lineNumber + "]"));
+    }
+
+    public void testAddingDoWithHeaderWithoutSkipHeaders() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCallSection = new ApiCallSection("test");
+        apiCallSection.addHeaders(Collections.singletonMap("header", "value"));
+        doSection.setApiCallSection(apiCallSection);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [headers] section without a corresponding " +
+            "[\"skip\": \"features\": \"headers\"] so runners that do not support the [headers] section can skip the test at line ["
+            + lineNumber + "]"));
+    }
+
+    public void testAddingDoWithNodeSelectorWithoutSkipNodeSelector() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCall = new ApiCallSection("test");
+        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+        doSection.setApiCallSection(apiCall);
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [do] with a [node_selector] section without a " +
+            "corresponding [\"skip\": \"features\": \"node_selector\"] so runners that do not support the [node_selector] section can " +
+            "skip the test at line [" + lineNumber + "]"));
+    }
+
+    public void testAddingContainsWithoutSkipContains() {
+        int lineNumber = between(1, 10000);
+        ContainsAssertion containsAssertion = new ContainsAssertion(
+            new XContentLocation(lineNumber, 0),
+            randomAlphaOfLength(randomIntBetween(3, 30)),
+            randomDouble());
+        ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, containsAssertion);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertThat(e.getMessage(), containsString("api/name:\nattempted to add a [contains] assertion without a corresponding " +
+            "[\"skip\": \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
+            "can skip the test at line [" + lineNumber + "]"));
+    }
+
+    public void testMultipleValidationErrors() {
+        int firstLineNumber = between(1, 10000);
+        List<ClientYamlTestSection> sections = new ArrayList<>();
+        {
+            ContainsAssertion containsAssertion = new ContainsAssertion(
+                new XContentLocation(firstLineNumber, 0),
+                randomAlphaOfLength(randomIntBetween(3, 30)),
+                randomDouble());
+            sections.add(new ClientYamlTestSection(
+                new XContentLocation(0, 0), "section1", SkipSection.EMPTY, Collections.singletonList(containsAssertion)));
+        }
+        int secondLineNumber = between(1, 10000);
+        int thirdLineNumber = between(1, 10000);
+        List<ExecutableSection> doSections = new ArrayList<>();
+        {
+            DoSection doSection = new DoSection(new XContentLocation(secondLineNumber, 0));
+            doSection.setExpectedWarningHeaders(singletonList("foo"));
+            doSection.setApiCallSection(new ApiCallSection("test"));
+            doSections.add(doSection);
+        }
+        {
+            DoSection doSection = new DoSection(new XContentLocation(thirdLineNumber, 0));
+            ApiCallSection apiCall = new ApiCallSection("test");
+            apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+            doSection.setApiCallSection(apiCall);
+            doSections.add(doSection);
+        }
+        sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", SkipSection.EMPTY, doSections));
+
+        ClientYamlTestSuite testSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY, sections);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertEquals("api/name:\n" +
+            "attempted to add a [contains] assertion without a corresponding [\"skip\": \"features\": \"contains\"] so runners " +
+                "that do not support the [contains] assertion can skip the test at line [" + firstLineNumber + "],\n" +
+            "attempted to add a [do] with a [warnings] section without a corresponding [\"skip\": \"features\": \"warnings\"] so " +
+                "runners that do not support the [warnings] section can skip the test at line [" + secondLineNumber + "],\n" +
+            "attempted to add a [do] with a [node_selector] section without a corresponding [\"skip\": \"features\": \"node_selector\"] " +
+                "so runners that do not support the [node_selector] section can skip the test at line [" + thirdLineNumber + "]",
+            e.getMessage());
+    }
+
+    public void testAddingDoWithWarningWithSkip() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        SkipSection skipSection = new SkipSection(null, singletonList("warnings"), null);
+        createTestSuite(skipSection, doSection).validate();
+    }
+
+    public void testAddingDoWithNodeSelectorWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("node_selector"), null);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCall = new ApiCallSection("test");
+        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+        doSection.setApiCallSection(apiCall);
+        createTestSuite(skipSection, doSection).validate();
+    }
+
+    public void testAddingDoWithHeadersWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("headers"), null);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCallSection = new ApiCallSection("test");
+        apiCallSection.addHeaders(singletonMap("foo", "bar"));
+        doSection.setApiCallSection(apiCallSection);
+        createTestSuite(skipSection, doSection).validate();
+    }
+
+    public void testAddingContainsWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("contains"), null);
+        ContainsAssertion containsAssertion = new ContainsAssertion(
+            new XContentLocation(lineNumber, 0),
+            randomAlphaOfLength(randomIntBetween(3, 30)),
+            randomDouble());
+        createTestSuite(skipSection, containsAssertion).validate();
+    }
+
+    private static ClientYamlTestSuite createTestSuite(SkipSection skipSection, ExecutableSection executableSection) {
+        final SetupSection setupSection;
+        final TeardownSection teardownSection;
+        final ClientYamlTestSection clientYamlTestSection;
+        switch(randomIntBetween(0, 4)) {
+            case 0:
+                setupSection = new SetupSection(skipSection, Collections.emptyList());
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.singletonList(executableSection));
+                break;
+            case 1:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = new TeardownSection(skipSection, Collections.emptyList());
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.singletonList(executableSection));
+                break;
+            case 2:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    skipSection, Collections.singletonList(executableSection));
+                break;
+            case 3:
+                setupSection = new SetupSection(skipSection, Collections.singletonList(executableSection));
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection));
+                break;
+            case 4:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = new TeardownSection(skipSection, Collections.singletonList(executableSection));
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection));
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+        return new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
+            Collections.singletonList(clientYamlTestSection));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index 6df564afeff44..5a201c66aa1ce 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -22,6 +22,7 @@
 import org.apache.http.HttpHost;
 import org.elasticsearch.client.Node;
 import org.elasticsearch.client.NodeSelector;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentLocation;
@@ -52,7 +53,7 @@
 
 public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
 
-    public void testWarningHeaders() throws IOException {
+    public void testWarningHeaders() {
         {
             final DoSection section = new DoSection(new XContentLocation(1, 1));
 
@@ -424,6 +425,17 @@ public void testParseDoSectionWithCatch() throws Exception {
         assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
     }
 
+    public void testUnsupportedTopLevelField() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "max_concurrent_shard_requests: 1"
+        );
+
+        ParsingException e = expectThrows(ParsingException.class, () -> DoSection.parse(parser));
+        assertThat(e.getMessage(), is("unsupported field [max_concurrent_shard_requests]"));
+        parser.nextToken();
+        parser.nextToken();
+    }
+
     public void testParseDoSectionWithHeaders() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "headers:\n" +
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
index 15ca1ec0096e3..3d4d5140632c1 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
@@ -49,8 +49,8 @@ public void testParseTeardownSection() throws Exception {
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(true));
         assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+        assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));
     }
 
     public void testParseWithSkip() throws Exception {
@@ -79,7 +79,7 @@ public void testParseWithSkip() throws Exception {
         assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_5_3_0));
         assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
         assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+        assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
index 90c7421108974..2c031d9e3acff 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.test.test;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
diff --git a/x-pack/build.gradle b/x-pack/build.gradle
index 0b1406519685b..e1c72c734798e 100644
--- a/x-pack/build.gradle
+++ b/x-pack/build.gradle
@@ -34,6 +34,7 @@ subprojects {
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-core:${version}": xpackModule('core')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-deprecation:${version}": xpackModule('deprecation')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-graph:${version}": xpackModule('graph')]
+  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ilm:${version}": xpackModule('ilm')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-logstash:${version}": xpackModule('logstash')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ml:${version}": xpackModule('ml')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-monitoring:${version}": xpackModule('monitoring')]
diff --git a/x-pack/docs/en/rest-api/watcher/stats.asciidoc b/x-pack/docs/en/rest-api/watcher/stats.asciidoc
index d4bfc9de70789..3f875485ba006 100644
--- a/x-pack/docs/en/rest-api/watcher/stats.asciidoc
+++ b/x-pack/docs/en/rest-api/watcher/stats.asciidoc
@@ -25,7 +25,7 @@ currently being executed by {watcher}. Additional information is shared per
 watch that is currently executing. This information includes the `watch_id`,
 the time its execution started and its current execution phase.
 
-To include this metric, the `metric` option should be set to `pending_watches`
+To include this metric, the `metric` option should be set to `current_watches`
 or `_all`. In addition you can also specify the `emit_stacktraces=true`
 parameter, which adds stack traces for each watch that is being executed. These
 stack traces can give you more insight into an execution of a watch.
@@ -51,7 +51,7 @@ To include this metric, the `metric` option should include `queued_watches` or
 
 `metric`::
   (enum) Defines which additional metrics are included in the response.
-  `pending_watches`::: Includes the current executing watches in the response.
+  `current_watches`::: Includes the current executing watches in the response.
   `queued_watches`::: Includes the watches queued for execution in the response.
   `_all`::: Includes all metrics in the response.
 
@@ -98,7 +98,7 @@ and will include the basic metrics and metrics about the current executing watch
 
 [source,js]
 --------------------------------------------------
-GET _xpack/watcher/stats?metric=pending_watches
+GET _xpack/watcher/stats?metric=current_watches
 --------------------------------------------------
 // CONSOLE
 
diff --git a/x-pack/docs/en/security/authorization/set-security-user.asciidoc b/x-pack/docs/en/security/authorization/set-security-user.asciidoc
index 92b9ae275aec8..c0f28ec0465e8 100644
--- a/x-pack/docs/en/security/authorization/set-security-user.asciidoc
+++ b/x-pack/docs/en/security/authorization/set-security-user.asciidoc
@@ -9,53 +9,18 @@ To guarantee that a user reads only their own documents, it makes sense to set u
 document level security. In this scenario, each document must have the username
 or role name associated with it, so that this information can be used by the
 role query for document level security. This is a situation where the
-`set_security_user` ingest processor can help.
+{ref}/ingest-node-set-security-user-processor.html[Set Security User Processor] ingest processor can help.
 
 NOTE: Document level security doesn't apply to write APIs. You must use unique
 ids for each user that uses the same index, otherwise they might overwrite other
 users' documents. The ingest processor just adds properties for the current
 authenticated user to the documents that are being indexed.
 
-The `set_security_user` processor attaches user-related details (such as
+The {ref}/ingest-node-set-security-user-processor.html[set security user processor] attaches user-related details (such as
 `username`,  `roles`, `email`, `full_name` and `metadata` ) from the current
 authenticated user to the current document by pre-processing the ingest. When
 you index data with an ingest pipeline, user details are automatically attached
-to the document. For example:
+to the document.
 
-[source,js]
---------------------------------------------------
-PUT shared-logs/log/1?pipeline=my_pipeline_id
-{
-  ...
-}
---------------------------------------------------
-// NOTCONSOLE
+For more information see {ref}/ingest.html[Ingest node] and {ref}/ingest-node-set-security-user-processor.html[Set security user processor].
 
-For more information about setting up a pipeline and other processors, see 
-{ref}/ingest.html[ingest node].
-
-[[set-security-user-options]]
-.Set Security User Options
-[options="header"]
-|======
-| Name          | Required  | Default                                                   | Description
-| `field`       | yes       | -                                                         | The field to store the user information into.
-| `properties`  | no        | [`username`, `roles`, `email`, `full_name`, `metadata`]   | Controls what user related properties are added to the `field`.
-|======
-
-The following example adds all user details for the current authenticated user
-to the `user` field for all documents that are processed by this pipeline:
-
-[source,js]
---------------------------------------------------
-{
-  "processors" : [
-    {
-      "set_security_user": {
-        "field": "user"
-      }
-    }
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
\ No newline at end of file
diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle
index f408e6a78b603..d3e95d997c3fb 100644
--- a/x-pack/plugin/ccr/qa/build.gradle
+++ b/x-pack/plugin/ccr/qa/build.gradle
@@ -15,14 +15,3 @@ subprojects {
     }
   }
 }
-
-/* Remove assemble on all qa projects because we don't need to publish
- * artifacts for them. */
-gradle.projectsEvaluated {
-  subprojects {
-    Task assemble = project.tasks.findByName('assemble')
-    if (assemble) {
-      assemble.enabled = false
-    }
-  }
-}
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
index 68c80b09a984e..b178ab2c562fc 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
@@ -156,7 +156,7 @@ public Collection<Object> createComponents(
     @Override
     public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterService clusterService,
                                                                        ThreadPool threadPool, Client client) {
-        return Collections.singletonList(new ShardFollowTasksExecutor(settings, client, threadPool, clusterService));
+        return Collections.singletonList(new ShardFollowTasksExecutor(client, threadPool, clusterService));
     }
 
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index 72793b4bd8e36..0519596e3d6c2 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -6,6 +6,8 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -26,6 +28,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardNotStartedException;
@@ -33,6 +36,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -376,6 +380,21 @@ protected void asyncShardOperation(
             }
         }
 
+        @Override
+        protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
+            ActionListener<Response> wrappedListener = ActionListener.wrap(listener::onResponse, e -> {
+                Throwable cause = ExceptionsHelper.unwrapCause(e);
+                if (cause instanceof IllegalStateException && cause.getMessage().contains("Not all operations between from_seqno [")) {
+                    String message = "Operations are no longer available for replicating. Maybe increase the retention setting [" +
+                        IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey() + "]?";
+                    listener.onFailure(new ElasticsearchException(message, e));
+                } else {
+                    listener.onFailure(e);
+                }
+            });
+            super.doExecute(task, request, wrappedListener);
+        }
+
         private void globalCheckpointAdvanced(
                 final ShardId shardId,
                 final long globalCheckpoint,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
index 8c302344ad86d..13dc736722e99 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.NodeDisconnectedException;
@@ -416,6 +417,7 @@ static boolean shouldRetry(Exception e) {
             actual instanceof IndexClosedException || // If follow index is closed
             actual instanceof NodeDisconnectedException ||
             actual instanceof NodeNotConnectedException ||
+            actual instanceof NodeClosedException ||
             (actual.getMessage() != null && actual.getMessage().contains("TransportService is closed"));
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
index 88d07566c74bd..446e3aaee41d3 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -58,8 +57,8 @@ public class ShardFollowTasksExecutor extends PersistentTasksExecutor<ShardFollo
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
 
-    public ShardFollowTasksExecutor(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) {
-        super(settings, ShardFollowTask.NAME, Ccr.CCR_THREAD_POOL_NAME);
+    public ShardFollowTasksExecutor(Client client, ThreadPool threadPool, ClusterService clusterService) {
+        super(ShardFollowTask.NAME, Ccr.CCR_THREAD_POOL_NAME);
         this.client = client;
         this.threadPool = threadPool;
         this.clusterService = clusterService;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
index a0a70c3dedfa2..b3cb9bdc87fc2 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.tasks.Task;
@@ -28,6 +29,7 @@
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 
+import java.io.IOException;
 import java.util.Objects;
 
 public class TransportCcrStatsAction extends TransportMasterNodeAction<CcrStatsAction.Request, CcrStatsAction.Response> {
@@ -55,8 +57,8 @@ public TransportCcrStatsAction(
             clusterService,
             threadPool,
             actionFilters,
-            indexNameExpressionResolver,
-            CcrStatsAction.Request::new
+            CcrStatsAction.Request::new,
+            indexNameExpressionResolver
         );
         this.client = client;
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
@@ -70,7 +72,12 @@ protected String executor() {
 
     @Override
     protected CcrStatsAction.Response newResponse() {
-        return new CcrStatsAction.Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected CcrStatsAction.Response read(StreamInput in) throws IOException {
+        return new CcrStatsAction.Response(in);
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
index 5d5885294f357..746700e2861d7 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
@@ -37,7 +37,7 @@ public TransportDeleteAutoFollowPatternAction(Settings settings, TransportServic
                                                   ThreadPool threadPool, ActionFilters actionFilters,
                                                   IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, DeleteAutoFollowPatternAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            indexNameExpressionResolver, DeleteAutoFollowPatternAction.Request::new);
+            DeleteAutoFollowPatternAction.Request::new, indexNameExpressionResolver);
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
index 86cae048efba1..32c8f65a57c62 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -24,6 +25,7 @@
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
 import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
@@ -38,7 +40,7 @@ public TransportGetAutoFollowPatternAction(Settings settings,
                                                ActionFilters actionFilters,
                                                IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, GetAutoFollowPatternAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            indexNameExpressionResolver, GetAutoFollowPatternAction.Request::new);
+            GetAutoFollowPatternAction.Request::new, indexNameExpressionResolver);
     }
 
     @Override
@@ -48,7 +50,12 @@ protected String executor() {
 
     @Override
     protected GetAutoFollowPatternAction.Response newResponse() {
-        return new GetAutoFollowPatternAction.Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected GetAutoFollowPatternAction.Response read(StreamInput in) throws IOException {
+        return new GetAutoFollowPatternAction.Response(in);
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
index 0ad268cce3e49..5caa06f56c45e 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
@@ -40,7 +40,7 @@ public TransportPauseFollowAction(
             final ClusterService clusterService,
             final PersistentTasksService persistentTasksService) {
         super(settings, PauseFollowAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            indexNameExpressionResolver, PauseFollowAction.Request::new);
+            PauseFollowAction.Request::new, indexNameExpressionResolver);
         this.persistentTasksService = persistentTasksService;
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
index 8416c414990df..d61080f68fb44 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
@@ -53,7 +53,7 @@ public TransportPutAutoFollowPatternAction(
             final IndexNameExpressionResolver indexNameExpressionResolver,
             final CcrLicenseChecker ccrLicenseChecker) {
         super(settings, PutAutoFollowPatternAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            indexNameExpressionResolver, PutAutoFollowPatternAction.Request::new);
+            PutAutoFollowPatternAction.Request::new, indexNameExpressionResolver);
         this.client = client;
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker");
     }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
index cdf496cc03287..4c8dcf226977a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.license.LicenseUtils;
@@ -38,6 +39,7 @@
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -68,11 +70,11 @@ public TransportPutFollowAction(
                 clusterService,
                 threadPool,
                 actionFilters,
-                indexNameExpressionResolver,
-                PutFollowAction.Request::new);
+                PutFollowAction.Request::new,
+                indexNameExpressionResolver);
         this.client = client;
         this.allocationService = allocationService;
-        this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.activeShardsObserver = new ActiveShardsObserver(clusterService, threadPool);
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
     }
 
@@ -83,7 +85,12 @@ protected String executor() {
 
     @Override
     protected PutFollowAction.Response newResponse() {
-        return new PutFollowAction.Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected PutFollowAction.Response read(StreamInput in) throws IOException {
+        return new PutFollowAction.Response(in);
     }
 
     @Override
@@ -128,6 +135,7 @@ private void createFollowerIndex(
         if (leaderIndexMetaData.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
             listener.onFailure(
                 new IllegalArgumentException("leader index [" + request.getLeaderIndex() + "] does not have soft deletes enabled"));
+            return;
         }
 
         ActionListener<Boolean> handler = ActionListener.wrap(
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
index 7a9012ea26164..ad75f487089b5 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
@@ -8,10 +8,12 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@@ -50,9 +52,9 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public class TransportResumeFollowAction extends HandledTransportAction<ResumeFollowAction.Request, AcknowledgedResponse> {
+public class TransportResumeFollowAction extends TransportMasterNodeAction<ResumeFollowAction.Request, AcknowledgedResponse> {
 
-    static final ByteSizeValue DEFAULT_MAX_READ_REQUEST_SIZE = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES);
+    static final ByteSizeValue DEFAULT_MAX_READ_REQUEST_SIZE = new ByteSizeValue(32, ByteSizeUnit.MB);
     static final ByteSizeValue DEFAULT_MAX_WRITE_REQUEST_SIZE = new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES);
     private static final TimeValue DEFAULT_MAX_RETRY_DELAY = new TimeValue(500);
     private static final int DEFAULT_MAX_OUTSTANDING_WRITE_REQUESTS = 9;
@@ -64,8 +66,6 @@ public class TransportResumeFollowAction extends HandledTransportAction<ResumeFo
     static final TimeValue DEFAULT_READ_POLL_TIMEOUT = TimeValue.timeValueMinutes(1);
 
     private final Client client;
-    private final ThreadPool threadPool;
-    private final ClusterService clusterService;
     private final PersistentTasksService persistentTasksService;
     private final IndicesService indicesService;
     private final CcrLicenseChecker ccrLicenseChecker;
@@ -82,25 +82,38 @@ public TransportResumeFollowAction(
             final PersistentTasksService persistentTasksService,
             final IndicesService indicesService,
             final CcrLicenseChecker ccrLicenseChecker) {
-        super(settings, ResumeFollowAction.NAME, threadPool, transportService, actionFilters,
-            indexNameExpressionResolver, ResumeFollowAction.Request::new);
+        super(settings, ResumeFollowAction.NAME, true, transportService, clusterService, threadPool, actionFilters,
+            ResumeFollowAction.Request::new, indexNameExpressionResolver);
         this.client = client;
-        this.threadPool = threadPool;
-        this.clusterService = clusterService;
         this.persistentTasksService = persistentTasksService;
         this.indicesService = indicesService;
         this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
     }
 
     @Override
-    protected void doExecute(final ResumeFollowAction.Request request,
-                             final ActionListener<AcknowledgedResponse> listener) {
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(ResumeFollowAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(final ResumeFollowAction.Request request,
+                                   ClusterState state,
+                                   final ActionListener<AcknowledgedResponse> listener) throws Exception {
         if (ccrLicenseChecker.isCcrAllowed() == false) {
             listener.onFailure(LicenseUtils.newComplianceException("ccr"));
             return;
         }
 
-        final ClusterState state = clusterService.state();
         final IndexMetaData followerIndexMetadata = state.getMetaData().index(request.getFollowerIndex());
         if (followerIndexMetadata == null) {
             listener.onFailure(new IndexNotFoundException(request.getFollowerIndex()));
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
index 818a62cfe53a3..3a8884276eb04 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -34,7 +35,7 @@ public TransportUnfollowAction(Settings settings, TransportService transportServ
                                    ThreadPool threadPool, ActionFilters actionFilters,
                                    IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, UnfollowAction.NAME, transportService, clusterService, threadPool, actionFilters,
-            indexNameExpressionResolver, UnfollowAction.Request::new);
+            UnfollowAction.Request::new, indexNameExpressionResolver);
     }
 
     @Override
@@ -78,6 +79,18 @@ protected ClusterBlockException checkBlock(UnfollowAction.Request request, Clust
 
     static ClusterState unfollow(String followerIndex, ClusterState current) {
         IndexMetaData followerIMD = current.metaData().index(followerIndex);
+        if (followerIMD == null) {
+            throw new IndexNotFoundException(followerIndex);
+        }
+
+        if (followerIMD.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY) == null) {
+            throw new IllegalArgumentException("index [" + followerIndex + "] is not a follower index");
+        }
+
+        if (followerIMD.getState() != IndexMetaData.State.CLOSE) {
+            throw new IllegalArgumentException("cannot convert the follower index [" + followerIndex +
+                "] to a non-follower, because it has not been closed");
+        }
 
         PersistentTasksCustomMetaData persistentTasks = current.metaData().custom(PersistentTasksCustomMetaData.TYPE);
         if (persistentTasks != null) {
@@ -92,11 +105,6 @@ static ClusterState unfollow(String followerIndex, ClusterState current) {
             }
         }
 
-        if (followerIMD.getState() != IndexMetaData.State.CLOSE) {
-            throw new IllegalArgumentException("cannot convert the follower index [" + followerIndex +
-                "] to a non-follower, because it has not been closed");
-        }
-
         IndexMetaData.Builder newIMD = IndexMetaData.builder(followerIMD);
         // Remove index.xpack.ccr.following_index setting
         Settings.Builder builder = Settings.builder();
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
index 72c44a48f985e..9bfd260ce63b4 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java
@@ -31,8 +31,7 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        Request request = new Request();
-        request.setName(restRequest.param("name"));
+        Request request = new Request(restRequest.param("name"));
         return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index 6fe7b0fd75c01..07a4dc44cd20f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
@@ -25,9 +26,11 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -38,6 +41,8 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.engine.DocIdSeqNoAndTerm;
+import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardTestCase;
@@ -76,12 +81,15 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
@@ -90,6 +98,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public abstract class CcrIntegTestCase extends ESTestCase {
@@ -240,10 +249,12 @@ protected final ClusterHealthStatus ensureLeaderYellow(String... indices) {
     }
 
     protected final ClusterHealthStatus ensureLeaderGreen(String... indices) {
+        logger.info("ensure green leader indices {}", Arrays.toString(indices));
         return ensureColor(clusterGroup.leaderCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), false, indices);
     }
 
     protected final ClusterHealthStatus ensureFollowerGreen(String... indices) {
+        logger.info("ensure green follower indices {}", Arrays.toString(indices));
         return ensureColor(clusterGroup.followerCluster, ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(30), false, indices);
     }
 
@@ -391,20 +402,54 @@ public static ResumeFollowAction.Request resumeFollow(String followerIndex) {
         return request;
     }
 
-    protected void assertSameDocCount(String leaderIndex, String followerIndex) throws Exception {
-        refresh(leaderClient(), leaderIndex);
-        SearchRequest request1 = new SearchRequest(leaderIndex);
-        request1.source(new SearchSourceBuilder().size(0));
-        SearchResponse response1 = leaderClient().search(request1).actionGet();
+    /**
+     * This asserts the index is fully replicated from the leader index to the follower index. It first verifies that the seq_no_stats
+     * on the follower equal the leader's; then verifies the existing pairs of (docId, seqNo) on the follower also equal the leader.
+     */
+    protected void assertIndexFullyReplicatedToFollower(String leaderIndex, String followerIndex) throws Exception {
+        logger.info("--> asserting seq_no_stats between {} and {}", leaderIndex, followerIndex);
         assertBusy(() -> {
-            refresh(followerClient(), followerIndex);
-            SearchRequest request2 = new SearchRequest(followerIndex);
-            request2.source(new SearchSourceBuilder().size(0));
-            SearchResponse response2 = followerClient().search(request2).actionGet();
-            assertThat(response2.getHits().getTotalHits(), equalTo(response1.getHits().getTotalHits()));
+            Map<Integer, SeqNoStats> leaderStats = new HashMap<>();
+            for (ShardStats shardStat : leaderClient().admin().indices().prepareStats(leaderIndex).clear().get().getShards()) {
+                if (shardStat.getSeqNoStats() == null) {
+                    throw new AssertionError("leader seq_no_stats is not available [" + Strings.toString(shardStat) + "]");
+                }
+                leaderStats.put(shardStat.getShardRouting().shardId().id(), shardStat.getSeqNoStats());
+            }
+            Map<Integer, SeqNoStats> followerStats = new HashMap<>();
+            for (ShardStats shardStat : followerClient().admin().indices().prepareStats(followerIndex).clear().get().getShards()) {
+                if (shardStat.getSeqNoStats() == null) {
+                    throw new AssertionError("follower seq_no_stats is not available [" + Strings.toString(shardStat) + "]");
+                }
+                followerStats.put(shardStat.getShardRouting().shardId().id(), shardStat.getSeqNoStats());
+            }
+            assertThat(leaderStats, equalTo(followerStats));
+        }, 60, TimeUnit.SECONDS);
+        logger.info("--> asserting <<docId,seqNo>> between {} and {}", leaderIndex, followerIndex);
+        assertBusy(() -> {
+            assertThat(getDocIdAndSeqNos(clusterGroup.leaderCluster, leaderIndex),
+                equalTo(getDocIdAndSeqNos(clusterGroup.followerCluster, followerIndex)));
         }, 60, TimeUnit.SECONDS);
     }
 
+    private Map<Integer, List<DocIdSeqNoAndTerm>> getDocIdAndSeqNos(InternalTestCluster cluster, String index) throws IOException {
+        final ClusterState state = cluster.client().admin().cluster().prepareState().get().getState();
+        List<ShardRouting> shardRoutings = state.routingTable().allShards(index);
+        Randomness.shuffle(shardRoutings);
+        final Map<Integer, List<DocIdSeqNoAndTerm>> docs = new HashMap<>();
+        for (ShardRouting shardRouting : shardRoutings) {
+            if (shardRouting == null || shardRouting.assignedToNode() == false || docs.containsKey(shardRouting.shardId().id())) {
+                continue;
+            }
+            IndexShard indexShard = cluster.getInstance(IndicesService.class, state.nodes().get(shardRouting.currentNodeId()).getName())
+                .indexServiceSafe(shardRouting.index()).getShard(shardRouting.id());
+            docs.put(shardRouting.shardId().id(), IndexShardTestCase.getDocIdAndSeqNos(indexShard).stream()
+                .map(d -> new DocIdSeqNoAndTerm(d.getId(), d.getSeqNo(), 1L))  // normalize primary term as the follower use its own term
+                .collect(Collectors.toList()));
+        }
+        return docs;
+    }
+
     protected void atLeastDocsIndexed(Client client, String index, long numDocsReplicated) throws InterruptedException {
         logger.info("waiting for at least [{}] documents to be indexed into index [{}]", numDocsReplicated, index);
         awaitBusy(() -> {
@@ -416,6 +461,20 @@ protected void atLeastDocsIndexed(Client client, String index, long numDocsRepli
         }, 60, TimeUnit.SECONDS);
     }
 
+    protected void awaitGlobalCheckpointAtLeast(Client client, ShardId shardId, long minimumGlobalCheckpoint) throws Exception {
+        logger.info("waiting for the global checkpoint on [{}] at least [{}]", shardId, minimumGlobalCheckpoint);
+        assertBusy(() -> {
+            ShardStats stats = client.admin().indices().prepareStats(shardId.getIndexName()).clear().get()
+                .asMap().entrySet().stream().filter(e -> e.getKey().shardId().equals(shardId))
+                .map(Map.Entry::getValue).findFirst().orElse(null);
+            if (stats == null || stats.getSeqNoStats() == null) {
+                throw new AssertionError("seq_no_stats for shard [" + shardId + "] is not found"); // causes assertBusy to retry
+            }
+            assertThat(Strings.toString(stats.getSeqNoStats()),
+                stats.getSeqNoStats().getGlobalCheckpoint(), greaterThanOrEqualTo(minimumGlobalCheckpoint));
+        }, 60, TimeUnit.SECONDS);
+    }
+
     protected void assertMaxSeqNoOfUpdatesIsTransferred(Index leaderIndex, Index followerIndex, int numberOfShards) throws Exception {
         assertBusy(() -> {
             long[] msuOnLeader = new long[numberOfShards];
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index 8a94ba606b071..9f22dc320cbdd 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -254,8 +254,7 @@ private void putAutoFollowPatterns(String name, String[] patterns) {
     }
 
     private void deleteAutoFollowPatternSetting() {
-        DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request();
-        request.setName("my-pattern");
+        DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request("my-pattern");
         assertTrue(followerClient().execute(DeleteAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged());
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
index 52c190f95f53c..6685776e9805d 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
@@ -6,22 +6,27 @@
 
 package org.elasticsearch.xpack.ccr;
 
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.node.NodeClosedException;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.xpack.CcrIntegTestCase;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 
 import java.util.Locale;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -45,23 +50,32 @@ public void testFailOverOnFollower() throws Exception {
         AtomicBoolean stopped = new AtomicBoolean();
         Thread[] threads = new Thread[between(1, 8)];
         AtomicInteger docID = new AtomicInteger();
+        Semaphore availableDocs = new Semaphore(0);
         for (int i = 0; i < threads.length; i++) {
             threads[i] = new Thread(() -> {
                 while (stopped.get() == false) {
                     try {
-                        if (frequently()) {
-                            String id = Integer.toString(frequently() ? docID.incrementAndGet() : between(0, 10)); // sometimes update
-                            leaderClient().prepareIndex("leader-index", "doc", id).setSource("{\"f\":" + id + "}", XContentType.JSON).get();
-                        } else {
-                            String id = Integer.toString(between(0, docID.get()));
-                            leaderClient().prepareDelete("leader-index", "doc", id).get();
+                        if (availableDocs.tryAcquire(10, TimeUnit.MILLISECONDS) == false) {
+                            continue;
                         }
-                    } catch (NodeClosedException ignored) {
+                    } catch (InterruptedException e) {
+                        throw new AssertionError(e);
+                    }
+                    if (frequently()) {
+                        String id = Integer.toString(frequently() ? docID.incrementAndGet() : between(0, 10)); // sometimes update
+                        IndexResponse indexResponse = leaderClient().prepareIndex("leader-index", "doc", id)
+                            .setSource("{\"f\":" + id + "}", XContentType.JSON).get();
+                        logger.info("--> index id={} seq_no={}", indexResponse.getId(), indexResponse.getSeqNo());
+                    } else {
+                        String id = Integer.toString(between(0, docID.get()));
+                        DeleteResponse deleteResponse = leaderClient().prepareDelete("leader-index", "doc", id).get();
+                        logger.info("--> delete id={} seq_no={}", deleteResponse.getId(), deleteResponse.getSeqNo());
                     }
                 }
             });
             threads[i].start();
         }
+        availableDocs.release(between(100, 200));
         PutFollowAction.Request follow = putFollow("leader-index", "follower-index");
         follow.getFollowRequest().setMaxReadRequestOperationCount(randomIntBetween(32, 2048));
         follow.getFollowRequest().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB));
@@ -69,9 +83,10 @@ public void testFailOverOnFollower() throws Exception {
         follow.getFollowRequest().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048));
         follow.getFollowRequest().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB));
         follow.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10));
+        logger.info("--> follow params {}", Strings.toString(follow.getFollowRequest()));
         followerClient().execute(PutFollowAction.INSTANCE, follow).get();
         ensureFollowerGreen("follower-index");
-        atLeastDocsIndexed(followerClient(), "follower-index", between(20, 60));
+        awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), between(30, 80));
         final ClusterState clusterState = getFollowerCluster().clusterService().state();
         for (ShardRouting shardRouting : clusterState.routingTable().allShards("follower-index")) {
             if (shardRouting.primary()) {
@@ -80,13 +95,15 @@ public void testFailOverOnFollower() throws Exception {
                 break;
             }
         }
+        availableDocs.release(between(50, 200));
         ensureFollowerGreen("follower-index");
-        atLeastDocsIndexed(followerClient(), "follower-index", between(80, 150));
+        availableDocs.release(between(50, 200));
+        awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), between(100, 150));
         stopped.set(true);
         for (Thread thread : threads) {
             thread.join();
         }
-        assertSameDocCount("leader-index", "follower-index");
+        assertIndexFullyReplicatedToFollower("leader-index", "follower-index");
         pauseFollow("follower-index");
     }
 
@@ -97,18 +114,23 @@ public void testFollowIndexAndCloseNode() throws Exception {
         ensureLeaderGreen("index1");
 
         AtomicBoolean run = new AtomicBoolean(true);
+        Semaphore availableDocs = new Semaphore(0);
         Thread thread = new Thread(() -> {
             int counter = 0;
             while (run.get()) {
-                final String source = String.format(Locale.ROOT, "{\"f\":%d}", counter++);
                 try {
-                    leaderClient().prepareIndex("index1", "doc")
-                        .setSource(source, XContentType.JSON)
-                        .setTimeout(TimeValue.timeValueSeconds(1))
-                        .get();
-                } catch (Exception e) {
-                    logger.error("Error while indexing into leader index", e);
+                    if (availableDocs.tryAcquire(10, TimeUnit.MILLISECONDS) == false) {
+                        continue;
+                    }
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
                 }
+                final String source = String.format(Locale.ROOT, "{\"f\":%d}", counter++);
+                IndexResponse indexResp = leaderClient().prepareIndex("index1", "doc")
+                    .setSource(source, XContentType.JSON)
+                    .setTimeout(TimeValue.timeValueSeconds(1))
+                    .get();
+                logger.info("--> index id={} seq_no={}", indexResp.getId(), indexResp.getSeqNo());
             }
         });
         thread.start();
@@ -121,19 +143,19 @@ public void testFollowIndexAndCloseNode() throws Exception {
         followRequest.getFollowRequest().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB));
         followRequest.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10));
         followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+        logger.info("--> follow params {}", Strings.toString(followRequest.getFollowRequest()));
 
-        long maxNumDocsReplicated = Math.min(1000, randomLongBetween(followRequest.getFollowRequest().getMaxReadRequestOperationCount(),
-            followRequest.getFollowRequest().getMaxReadRequestOperationCount() * 10));
-        long minNumDocsReplicated = maxNumDocsReplicated / 3L;
-        logger.info("waiting for at least [{}] documents to be indexed and then stop a random data node", minNumDocsReplicated);
-        atLeastDocsIndexed(followerClient(), "index2", minNumDocsReplicated);
+        int maxOpsPerRead = followRequest.getFollowRequest().getMaxReadRequestOperationCount();
+        int maxNumDocsReplicated = Math.min(between(50, 500), between(maxOpsPerRead, maxOpsPerRead * 10));
+        availableDocs.release(maxNumDocsReplicated / 2 + 1);
+        atLeastDocsIndexed(followerClient(), "index2", maxNumDocsReplicated / 3);
         getFollowerCluster().stopRandomNonMasterNode();
-        logger.info("waiting for at least [{}] documents to be indexed", maxNumDocsReplicated);
-        atLeastDocsIndexed(followerClient(), "index2", maxNumDocsReplicated);
+        availableDocs.release(maxNumDocsReplicated / 2 + 1);
+        atLeastDocsIndexed(followerClient(), "index2", maxNumDocsReplicated * 2 / 3);
         run.set(false);
         thread.join();
 
-        assertSameDocCount("index1", "index2");
+        assertIndexFullyReplicatedToFollower("index1", "index2");
         pauseFollow("index2");
         assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), 3);
     }
@@ -184,15 +206,15 @@ public void testAddNewReplicasOnFollower() throws Exception {
             }
         });
         flushingOnFollower.start();
-        atLeastDocsIndexed(followerClient(), "follower-index", 50);
+        awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), 50);
         followerClient().admin().indices().prepareUpdateSettings("follower-index")
             .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas + 1).build()).get();
         ensureFollowerGreen("follower-index");
-        atLeastDocsIndexed(followerClient(), "follower-index", 100);
+        awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), 100);
         stopped.set(true);
         flushingOnFollower.join();
         indexingOnLeader.join();
-        assertSameDocCount("leader-index", "follower-index");
+        assertIndexFullyReplicatedToFollower("leader-index", "follower-index");
         pauseFollow("follower-index");
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index eff849bca7671..ffc76b5c58e31 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -58,6 +58,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -211,14 +212,23 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon
             @Override
             public void afterBulk(long executionId, BulkRequest request, Throwable failure) {}
         };
+        int bulkSize = between(1, 20);
         BulkProcessor bulkProcessor = BulkProcessor.builder(leaderClient(), listener)
-            .setBulkActions(100)
+            .setBulkActions(bulkSize)
             .setConcurrentRequests(4)
             .build();
         AtomicBoolean run = new AtomicBoolean(true);
+        Semaphore availableDocs = new Semaphore(0);
         Thread thread = new Thread(() -> {
             int counter = 0;
             while (run.get()) {
+                try {
+                    if (availableDocs.tryAcquire(10, TimeUnit.MILLISECONDS) == false) {
+                        continue;
+                    }
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
                 final String source = String.format(Locale.ROOT, "{\"f\":%d}", counter++);
                 IndexRequest indexRequest = new IndexRequest("index1", "doc")
                     .source(source, XContentType.JSON)
@@ -229,26 +239,28 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
         thread.start();
 
         // Waiting for some document being index before following the index:
-        int maxReadSize = randomIntBetween(128, 2048);
-        long numDocsIndexed = Math.min(3000 * 2, randomLongBetween(maxReadSize, maxReadSize * 10));
+        int maxOpsPerRead = randomIntBetween(10, 100);
+        int numDocsIndexed = Math.min(between(20, 300), between(maxOpsPerRead, maxOpsPerRead * 10));
+        availableDocs.release(numDocsIndexed / 2 + bulkSize);
         atLeastDocsIndexed(leaderClient(), "index1", numDocsIndexed / 3);
 
         PutFollowAction.Request followRequest = putFollow("index1", "index2");
-        followRequest.getFollowRequest().setMaxReadRequestOperationCount(maxReadSize);
-        followRequest.getFollowRequest().setMaxOutstandingReadRequests(randomIntBetween(2, 10));
-        followRequest.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(2, 10));
+        followRequest.getFollowRequest().setMaxReadRequestOperationCount(maxOpsPerRead);
+        followRequest.getFollowRequest().setMaxOutstandingReadRequests(randomIntBetween(1, 10));
+        followRequest.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10));
         followRequest.getFollowRequest().setMaxWriteBufferCount(randomIntBetween(1024, 10240));
         followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
-
+        availableDocs.release(numDocsIndexed * 2  + bulkSize);
         atLeastDocsIndexed(leaderClient(), "index1", numDocsIndexed);
         run.set(false);
         thread.join();
         assertThat(bulkProcessor.awaitClose(1L, TimeUnit.MINUTES), is(true));
 
-        assertSameDocCount("index1", "index2");
+        assertIndexFullyReplicatedToFollower("index1", "index2");
+        pauseFollow("index2");
+        leaderClient().admin().indices().prepareRefresh("index1").get();
         assertTotalNumberOfOptimizedIndexing(resolveFollowerIndex("index2"), numberOfShards,
             leaderClient().prepareSearch("index1").get().getHits().totalHits);
-        pauseFollow("index2");
         assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), numberOfShards);
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
index 3398cada2d0d8..c651cca5b6a71 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
@@ -13,11 +14,11 @@
 import static org.elasticsearch.xpack.ccr.action.AutoFollowStatsTests.randomReadExceptions;
 import static org.elasticsearch.xpack.ccr.action.StatsResponsesTests.createStatsResponse;
 
-public class AutoFollowStatsResponseTests extends AbstractStreamableTestCase<CcrStatsAction.Response> {
+public class AutoFollowStatsResponseTests extends AbstractWireSerializingTestCase<CcrStatsAction.Response> {
 
     @Override
-    protected CcrStatsAction.Response createBlankInstance() {
-        return new CcrStatsAction.Response();
+    protected Writeable.Reader<CcrStatsAction.Response> instanceReader() {
+        return CcrStatsAction.Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
index b993132bcfa6c..8b8cd534c814d 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
@@ -5,20 +5,19 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
 
-public class DeleteAutoFollowPatternRequestTests extends AbstractStreamableTestCase<DeleteAutoFollowPatternAction.Request> {
+public class DeleteAutoFollowPatternRequestTests extends AbstractWireSerializingTestCase<DeleteAutoFollowPatternAction.Request> {
 
     @Override
-    protected DeleteAutoFollowPatternAction.Request createBlankInstance() {
-        return new DeleteAutoFollowPatternAction.Request();
+    protected Writeable.Reader<DeleteAutoFollowPatternAction.Request> instanceReader() {
+        return DeleteAutoFollowPatternAction.Request::new;
     }
 
     @Override
     protected DeleteAutoFollowPatternAction.Request createTestInstance() {
-        DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request();
-        request.setName(randomAlphaOfLength(4));
-        return request;
+        return new DeleteAutoFollowPatternAction.Request(randomAlphaOfLength(4));
     }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
index f71da0e578297..5284dd0fa61c1 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
@@ -5,14 +5,15 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction;
 
-public class GetAutoFollowPatternRequestTests extends AbstractStreamableTestCase<GetAutoFollowPatternAction.Request> {
+public class GetAutoFollowPatternRequestTests extends AbstractWireSerializingTestCase<GetAutoFollowPatternAction.Request> {
 
     @Override
-    protected GetAutoFollowPatternAction.Request createBlankInstance() {
-        return new GetAutoFollowPatternAction.Request();
+    protected Writeable.Reader<GetAutoFollowPatternAction.Request> instanceReader() {
+        return GetAutoFollowPatternAction.Request::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
index c74afd6075c95..7130c830baa01 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
 import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction;
 
@@ -16,11 +17,11 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class GetAutoFollowPatternResponseTests extends AbstractStreamableTestCase<GetAutoFollowPatternAction.Response> {
+public class GetAutoFollowPatternResponseTests extends AbstractWireSerializingTestCase<GetAutoFollowPatternAction.Response> {
 
     @Override
-    protected GetAutoFollowPatternAction.Response createBlankInstance() {
-        return new GetAutoFollowPatternAction.Response();
+    protected Writeable.Reader<GetAutoFollowPatternAction.Response> instanceReader() {
+        return GetAutoFollowPatternAction.Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
index e8aa0ab1f66f9..47da4aee1677e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
@@ -6,11 +6,12 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction;
 
 import java.io.IOException;
@@ -21,7 +22,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class PutAutoFollowPatternRequestTests extends AbstractStreamableXContentTestCase<PutAutoFollowPatternAction.Request> {
+public class PutAutoFollowPatternRequestTests extends AbstractSerializingTestCase<PutAutoFollowPatternAction.Request> {
 
     @Override
     protected boolean supportsUnknownFields() {
@@ -34,8 +35,8 @@ protected PutAutoFollowPatternAction.Request doParseInstance(XContentParser pars
     }
 
     @Override
-    protected PutAutoFollowPatternAction.Request createBlankInstance() {
-        return new PutAutoFollowPatternAction.Request();
+    protected Writeable.Reader<PutAutoFollowPatternAction.Request> instanceReader() {
+        return PutAutoFollowPatternAction.Request::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
index 1385b383b940c..726a1c9893a50 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
@@ -5,17 +5,18 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 
 import java.io.IOException;
 
-public class PutFollowActionRequestTests extends AbstractStreamableXContentTestCase<PutFollowAction.Request> {
+public class PutFollowActionRequestTests extends AbstractSerializingTestCase<PutFollowAction.Request> {
 
     @Override
-    protected PutFollowAction.Request createBlankInstance() {
-        return new PutFollowAction.Request();
+    protected Writeable.Reader<PutFollowAction.Request> instanceReader() {
+        return PutFollowAction.Request::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
index 506a5e6ffbb22..71b0074307eb4 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
@@ -5,14 +5,15 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
 
-public class PutFollowActionResponseTests extends AbstractStreamableTestCase<PutFollowAction.Response> {
+public class PutFollowActionResponseTests extends AbstractWireSerializingTestCase<PutFollowAction.Response> {
 
     @Override
-    protected PutFollowAction.Response createBlankInstance() {
-        return new PutFollowAction.Response();
+    protected Writeable.Reader<PutFollowAction.Response> instanceReader() {
+        return PutFollowAction.Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
index f93fd5229a20b..3d3e869f53e8a 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
@@ -6,11 +6,12 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
 
 import java.io.IOException;
@@ -19,11 +20,11 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class ResumeFollowActionRequestTests extends AbstractStreamableXContentTestCase<ResumeFollowAction.Request> {
+public class ResumeFollowActionRequestTests extends AbstractSerializingTestCase<ResumeFollowAction.Request> {
 
     @Override
-    protected ResumeFollowAction.Request createBlankInstance() {
-        return new ResumeFollowAction.Request();
+    protected Writeable.Reader<ResumeFollowAction.Request> instanceReader() {
+        return ResumeFollowAction.Request::new;
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java
index 422122dcbe400..f66a673e19dfa 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesTests.java
@@ -5,6 +5,9 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -84,4 +87,33 @@ public void testGetOperationsBasedOnGlobalSequenceId() throws Exception {
         assertThat(operation.id(), equalTo("5"));
     }
 
+    public void testMissingOperations() {
+        client().admin().indices().prepareCreate("index")
+            .setSettings(Settings.builder()
+                .put("index.soft_deletes.enabled", true)
+                .put("index.soft_deletes.retention.operations", 0)
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0))
+            .get();
+
+        for (int i = 0; i < 16; i++) {
+            client().prepareIndex("index", "_doc", "1").setSource("{}", XContentType.JSON).get();
+            client().prepareDelete("index", "_doc", "1").get();
+        }
+        client().admin().indices().refresh(new RefreshRequest("index")).actionGet();
+        ForceMergeRequest forceMergeRequest = new ForceMergeRequest("index");
+        forceMergeRequest.maxNumSegments(1);
+        client().admin().indices().forceMerge(forceMergeRequest).actionGet();
+
+        ShardStats shardStats = client().admin().indices().prepareStats("index").get().getIndex("index").getShards()[0];
+        String historyUUID = shardStats.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY);
+        ShardChangesAction.Request request =  new ShardChangesAction.Request(shardStats.getShardRouting().shardId(), historyUUID);
+        request.setFromSeqNo(0L);
+        request.setMaxOperationCount(1);
+
+        Exception e = expectThrows(ElasticsearchException.class, () -> client().execute(ShardChangesAction.INSTANCE, request).actionGet());
+        assertThat(e.getMessage(), equalTo("Operations are no longer available for replicating. Maybe increase the retention setting " +
+            "[index.soft_deletes.retention.operations]?"));
+    }
+
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
index 3f6b2fcb0e3ef..5ef43fc05c81c 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java
@@ -56,8 +56,7 @@ public void testInnerDelete() {
                 new AutoFollowMetadata(existingAutoFollowPatterns, existingAlreadyFollowedIndexUUIDS, existingHeaders)))
             .build();
 
-        Request request = new Request();
-        request.setName("name1");
+        Request request = new Request("name1");
         AutoFollowMetadata result = TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState)
             .getMetaData()
             .custom(AutoFollowMetadata.TYPE);
@@ -86,8 +85,7 @@ public void testInnerDeleteDoesNotExist() {
                 new AutoFollowMetadata(existingAutoFollowPatterns, existingAlreadyFollowedIndexUUIDS, existingHeaders)))
             .build();
 
-        Request request = new Request();
-        request.setName("name2");
+        Request request = new Request("name2");
         Exception e = expectThrows(ResourceNotFoundException.class,
             () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState));
         assertThat(e.getMessage(), equalTo("auto-follow pattern [name2] is missing"));
@@ -98,8 +96,7 @@ public void testInnerDeleteNoAutoFollowMetadata() {
             .metaData(MetaData.builder())
             .build();
 
-        Request request = new Request();
-        request.setName("name1");
+        Request request = new Request("name1");
         Exception e = expectThrows(ResourceNotFoundException.class,
             () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState));
         assertThat(e.getMessage(), equalTo("auto-follow pattern [name1] is missing"));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java
index 4a201e37355a9..9b9d088eea332 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowActionTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.test.ESTestCase;
@@ -104,4 +105,35 @@ public void testUnfollowRunningShardFollowTasks() {
             equalTo("cannot convert the follower index [follow_index] to a non-follower, because it has not been paused"));
     }
 
+    public void testUnfollowMissingIndex() {
+        IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index")
+            .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .state(IndexMetaData.State.CLOSE)
+            .putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
+
+        ClusterState current = ClusterState.builder(new ClusterName("cluster_name"))
+            .metaData(MetaData.builder()
+                .put(followerIndex)
+                .build())
+            .build();
+        expectThrows(IndexNotFoundException.class, () -> TransportUnfollowAction.unfollow("another_index", current));
+    }
+
+    public void testUnfollowNoneFollowIndex() {
+        IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index")
+            .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .state(IndexMetaData.State.CLOSE);
+
+        ClusterState current = ClusterState.builder(new ClusterName("cluster_name"))
+            .metaData(MetaData.builder()
+                .put(followerIndex)
+                .build())
+            .build();
+        expectThrows(IllegalArgumentException.class, () -> TransportUnfollowAction.unfollow("follow_index", current));
+    }
+
 }
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index efc8715baa154..94a70c236a1b0 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -45,6 +45,7 @@ dependencies {
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
     testCompile project(path: ':modules:parent-join', configuration: 'runtime')
     testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
+    testCompile ("org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}")
 }
 
 ext.expansions = [
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index 0619aef6961cc..18ded8d585078 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -79,6 +79,8 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
     public static final long BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS =
             XPackInfoResponse.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS;
 
+    private final Settings settings;
+
     private final ClusterService clusterService;
 
     /**
@@ -118,6 +120,7 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste
     public LicenseService(Settings settings, ClusterService clusterService, Clock clock, Environment env,
                           ResourceWatcherService resourceWatcherService, XPackLicenseState licenseState) {
         super(settings);
+        this.settings = settings;
         this.clusterService = clusterService;
         this.clock = clock;
         this.scheduler = new SchedulerEngine(settings, clock);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
index 66fffbe28d397..3d69c4ad6c4ff 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
@@ -6,24 +6,88 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.xpack.common.ProtocolUtils;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
-import static org.elasticsearch.license.PostStartBasicResponse.Status.NEED_ACKNOWLEDGEMENT;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
-class PostStartBasicResponse extends AcknowledgedResponse {
+public class PostStartBasicResponse extends AcknowledgedResponse {
+
+    private static final ConstructingObjectParser<PostStartBasicResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "start_basic_response", true, (a, v) -> {
+        boolean basicWasStarted = (Boolean) a[0];
+        String errorMessage = (String) a[1];
+
+        if (basicWasStarted) {
+            return new PostStartBasicResponse(Status.GENERATED_BASIC);
+        }
+        Status status = Status.fromErrorMessage(errorMessage);
+        @SuppressWarnings("unchecked") Tuple<String, Map<String, String[]>> acknowledgements = (Tuple<String, Map<String, String[]>>) a[2];
+        return new PostStartBasicResponse(status, acknowledgements.v2(), acknowledgements.v1());
+    });
+
+    private static final ParseField BASIC_WAS_STARTED_FIELD = new ParseField("basic_was_started");
+    private static final ParseField ERROR_MESSAGE_FIELD = new ParseField("error_message");
+    private static final ParseField ACKNOWLEDGE_FIELD = new ParseField("acknowledge");
+    private static final ParseField MESSAGE_FIELD = new ParseField("message");
+
+    static {
+        PARSER.declareBoolean(constructorArg(), BASIC_WAS_STARTED_FIELD);
+        PARSER.declareString(optionalConstructorArg(), ERROR_MESSAGE_FIELD);
+        PARSER.declareObject(optionalConstructorArg(), (parser, v) -> {
+                Map<String, String[]> acknowledgeMessages = new HashMap<>();
+                String message = null;
+                XContentParser.Token token;
+                String currentFieldName = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else {
+                        if (currentFieldName == null) {
+                            throw new XContentParseException(parser.getTokenLocation(), "expected message header or acknowledgement");
+                        }
+                        if (MESSAGE_FIELD.getPreferredName().equals(currentFieldName)) {
+                            ensureExpectedToken(XContentParser.Token.VALUE_STRING, token, parser::getTokenLocation);
+                            message = parser.text();
+                        } else {
+                            if (token != XContentParser.Token.START_ARRAY) {
+                                throw new XContentParseException(parser.getTokenLocation(), "unexpected acknowledgement type");
+                            }
+                            List<String> acknowledgeMessagesList = new ArrayList<>();
+                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                ensureExpectedToken(XContentParser.Token.VALUE_STRING, token, parser::getTokenLocation);
+                                acknowledgeMessagesList.add(parser.text());
+                            }
+                            acknowledgeMessages.put(currentFieldName, acknowledgeMessagesList.toArray(new String[0]));
+                        }
+                    }
+                }
+                return new Tuple<>(message, acknowledgeMessages);
+            }, ACKNOWLEDGE_FIELD);
+    }
 
     private Map<String, String[]> acknowledgeMessages;
     private String acknowledgeMessage;
 
-    enum Status {
+    public enum Status {
         GENERATED_BASIC(true, null, RestStatus.OK),
         ALREADY_USING_BASIC(false, "Operation failed: Current license is basic.", RestStatus.FORBIDDEN),
         NEED_ACKNOWLEDGEMENT(false, "Operation failed: Needs acknowledgement.", RestStatus.OK);
@@ -49,19 +113,29 @@ String getErrorMessage() {
         RestStatus getRestStatus() {
             return restStatus;
         }
+
+        static Status fromErrorMessage(final String errorMessage) {
+            final Status[] values = Status.values();
+            for (Status status : values) {
+                if (Objects.equals(status.errorMessage, errorMessage)) {
+                    return status;
+                }
+            }
+            throw new IllegalArgumentException("No status for error message ['" + errorMessage + "']");
+        }
     }
 
     private Status status;
 
-    PostStartBasicResponse() {
+    public PostStartBasicResponse() {
     }
 
     PostStartBasicResponse(Status status) {
         this(status, Collections.emptyMap(), null);
     }
 
-    PostStartBasicResponse(Status status, Map<String, String[]> acknowledgeMessages, String acknowledgeMessage) {
-        super(status != NEED_ACKNOWLEDGEMENT);
+    public PostStartBasicResponse(Status status, Map<String, String[]> acknowledgeMessages, String acknowledgeMessage) {
+        super(status != Status.NEED_ACKNOWLEDGEMENT);
         this.status = status;
         this.acknowledgeMessages = acknowledgeMessages;
         this.acknowledgeMessage = acknowledgeMessage;
@@ -108,14 +182,14 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
         if (status.isBasicStarted()) {
-            builder.field("basic_was_started", true);
+            builder.field(BASIC_WAS_STARTED_FIELD.getPreferredName(), true);
         } else {
-            builder.field("basic_was_started", false);
-            builder.field("error_message", status.getErrorMessage());
+            builder.field(BASIC_WAS_STARTED_FIELD.getPreferredName(), false);
+            builder.field(ERROR_MESSAGE_FIELD.getPreferredName(), status.getErrorMessage());
         }
         if (acknowledgeMessages.isEmpty() == false) {
             builder.startObject("acknowledge");
-            builder.field("message", acknowledgeMessage);
+            builder.field(MESSAGE_FIELD.getPreferredName(), acknowledgeMessage);
             for (Map.Entry<String, String[]> entry : acknowledgeMessages.entrySet()) {
                 builder.startArray(entry.getKey());
                 for (String message : entry.getValue()) {
@@ -126,4 +200,26 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
             builder.endObject();
         }
     }
+
+    public static PostStartBasicResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        PostStartBasicResponse that = (PostStartBasicResponse) o;
+
+        return status == that.status &&
+            ProtocolUtils.equals(acknowledgeMessages, that.acknowledgeMessages) &&
+            Objects.equals(acknowledgeMessage, that.acknowledgeMessage);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), status, ProtocolUtils.hashCode(acknowledgeMessages), acknowledgeMessage);
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index f1bcc0b68310a..f5dab8ff8d4f8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.license;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.xpack.core.XPackField;
@@ -309,7 +309,7 @@ void update(OperationMode mode, boolean active, @Nullable Version mostRecentTria
                 // Before 6.3, Trial licenses would default having security enabled.
                 // If this license was generated before that version, then treat it as if security is explicitly enabled
                 if (mostRecentTrialVersion == null || mostRecentTrialVersion.before(Version.V_6_3_0)) {
-                    Loggers.getLogger(getClass()).info("Automatically enabling security for older trial license ({})",
+                    LogManager.getLogger(getClass()).info("Automatically enabling security for older trial license ({})",
                         mostRecentTrialVersion == null ? "[pre 6.1.0]" : mostRecentTrialVersion.toString());
                     isSecurityEnabledByTrialVersion = true;
                 }
@@ -605,6 +605,22 @@ public synchronized boolean isUpgradeAllowed() {
         return status.active;
     }
 
+    /**
+     * Determine if Index Lifecycle API should be enabled.
+     * <p>
+     * Index Lifecycle API is available in for all license types except
+     * {@link OperationMode#MISSING}
+     *
+     * @return {@code true} as long as the license is valid. Otherwise
+     *         {@code false}.
+     */
+    public boolean isIndexLifecycleAllowed() {
+        // status is volatile
+        Status localStatus = status;
+        // Should work on all active licenses
+        return localStatus.active;
+    }
+
     /**
      * Determine if SQL support should be enabled.
      * <p>
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java
index 12eb20617ff0f..b44e192f407ac 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponse.java
@@ -241,7 +241,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ShardSearchFailure.fromXContent(p), FAILURES);
     } 
     
-    public static GraphExploreResponse fromXContext(XContentParser parser) throws IOException {
+    public static GraphExploreResponse fromXContent(XContentParser parser) throws IOException {
         return PARSER.apply(parser, null);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
index dc4601ae82d5a..709d523216e07 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
@@ -197,7 +197,7 @@ private SegmentCommitInfo syncSegment(SegmentCommitInfo segmentCommitInfo, LiveD
             List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
             for (FieldInfo fieldInfo : fieldInfos) {
                 fieldInfoCopy.add(new FieldInfo(fieldInfo.name, fieldInfo.number,
-                    false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0,
+                    false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, fieldInfo.attributes(), 0, 0, 0,
                     fieldInfo.isSoftDeletesField()));
             }
             FieldInfos newFieldInfos = new FieldInfos(fieldInfoCopy.toArray(new FieldInfo[0]));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index c73bb8576a7ad..7dae8856921e8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -40,6 +40,7 @@ public final class ClientHelper {
     public static final String SECURITY_ORIGIN = "security";
     public static final String WATCHER_ORIGIN = "watcher";
     public static final String ML_ORIGIN = "ml";
+    public static final String INDEX_LIFECYCLE_ORIGIN = "index_lifecycle";
     public static final String MONITORING_ORIGIN = "monitoring";
     public static final String DEPRECATION_ORIGIN = "deprecation";
     public static final String PERSISTENT_TASK_ORIGIN = "persistent_tasks";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
index 3f27f66b27b77..db8981055d24d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackInfoRequestBuilder;
 import org.elasticsearch.xpack.core.ccr.client.CcrClient;
+import org.elasticsearch.xpack.core.indexlifecycle.client.ILMClient;
 import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.core.monitoring.client.MonitoringClient;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
@@ -36,6 +37,7 @@ public class XPackClient {
     private final SecurityClient securityClient;
     private final WatcherClient watcherClient;
     private final MachineLearningClient machineLearning;
+    private final ILMClient ilmClient;
 
     public XPackClient(Client client) {
         this.client = Objects.requireNonNull(client, "client");
@@ -45,6 +47,7 @@ public XPackClient(Client client) {
         this.securityClient = new SecurityClient(client);
         this.watcherClient = new WatcherClient(client);
         this.machineLearning = new MachineLearningClient(client);
+        this.ilmClient = new ILMClient(client);
     }
 
     public Client es() {
@@ -75,6 +78,10 @@ public MachineLearningClient machineLearning() {
         return machineLearning;
     }
 
+    public ILMClient ilmClient() {
+        return ilmClient;
+    }
+
     public XPackClient withHeaders(Map<String, String> headers) {
         return new XPackClient(client.filterWithHeader(headers));
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 480bd1f4534d3..203e696fb5f21 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -37,12 +37,30 @@
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
+import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
-import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -136,8 +154,8 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExceptExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
-import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
 import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
@@ -318,7 +336,15 @@ public List<GenericAction> getClientActions() {
                 StopRollupJobAction.INSTANCE,
                 DeleteRollupJobAction.INSTANCE,
                 GetRollupJobsAction.INSTANCE,
-                GetRollupCapsAction.INSTANCE
+                GetRollupCapsAction.INSTANCE,
+                // ILM
+                DeleteLifecycleAction.INSTANCE,
+                GetLifecycleAction.INSTANCE,
+                PutLifecycleAction.INSTANCE,
+                ExplainLifecycleAction.INSTANCE,
+                RemoveIndexLifecyclePolicyAction.INSTANCE,
+                MoveToStepAction.INSTANCE,
+                RetryAction.INSTANCE
         );
     }
 
@@ -372,9 +398,27 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(Task.Status.class, RollupJobStatus.NAME, RollupJobStatus::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskState.class, RollupJobStatus.NAME, RollupJobStatus::new),
                 // ccr
+                new NamedWriteableRegistry.Entry(AutoFollowMetadata.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
-                    in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in))
+                    in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
+                // ILM
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
+                    IndexLifecycleFeatureSetUsage::new),
+                // ILM - Custom Metadata
+                new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
+                new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
+                    IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new),
+                // ILM - LifecycleTypes
+                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                    (in) -> TimeseriesLifecycleType.INSTANCE),
+                // ILM - Lifecycle Actions
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new)
         );
     }
 
@@ -405,7 +449,7 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
                         RollupJobStatus::fromXContent),
                 new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(RollupJobStatus.NAME),
                         RollupJobStatus::fromXContent)
-        );
+            );
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
index 70eb047c8edef..0e6888dd80d73 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
@@ -31,6 +31,8 @@ public final class XPackField {
     public static final String SQL = "sql";
     /** Name constant for the rollup feature. */
     public static final String ROLLUP = "rollup";
+    /** Name constant for the index lifecycle feature. */
+    public static final String INDEX_LIFECYCLE = "ilm";
 
     private XPackField() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
index 809baae040e1b..89fda8cebd5a7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -248,7 +248,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         List<Object> components = new ArrayList<>();
 
         // just create the reloader as it will pull all of the loaded ssl configurations and start watching them
-        new SSLConfigurationReloader(settings, environment, getSslService(), resourceWatcherService);
+        new SSLConfigurationReloader(environment, getSslService(), resourceWatcherService);
 
         setLicenseService(new LicenseService(settings, clusterService, getClock(),
                 environment, resourceWatcherService, getLicenseState()));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index ac1dad2f1c7e4..9378b71b4b014 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -3,6 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 package org.elasticsearch.xpack.core;
 
 import org.elasticsearch.common.network.NetworkModule;
@@ -81,6 +82,12 @@ private XPackSettings() {
     public static final Setting<Boolean> BEATS_ENABLED = Setting.boolSetting("xpack.beats.enabled", true,
         Setting.Property.NodeScope);
 
+    /**
+     * Setting for enabling or disabling the index lifecycle extension. Defaults to true.
+     */
+    public static final Setting<Boolean> INDEX_LIFECYCLE_ENABLED = Setting.boolSetting("xpack.ilm.enabled", true,
+        Setting.Property.NodeScope);
+
     /** Setting for enabling or disabling TLS. Defaults to false. */
     public static final Setting<Boolean> TRANSPORT_SSL_ENABLED = Setting.boolSetting("xpack.security.transport.ssl.enabled", false,
             Property.NodeScope);
@@ -194,6 +201,7 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(USER_SETTING);
         settings.add(ROLLUP_ENABLED);
         settings.add(PASSWORD_HASHING_ALGORITHM);
+        settings.add(INDEX_LIFECYCLE_ENABLED);
         return Collections.unmodifiableList(settings);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index ef5909f49c388..bb9d9d90c2000 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
@@ -32,7 +33,12 @@ private CcrStatsAction() {
 
     @Override
     public Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> {
@@ -40,14 +46,13 @@ public static class Request extends MasterNodeRequest<Request> {
         public Request() {
         }
 
-        @Override
-        public ActionRequestValidationException validate() {
-            return null;
+        public Request(StreamInput in) throws IOException {
+            super(in);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public ActionRequestValidationException validate() {
+            return null;
         }
 
         @Override
@@ -58,15 +63,19 @@ public void writeTo(StreamOutput out) throws IOException {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private AutoFollowStats autoFollowStats;
-        private FollowStatsAction.StatsResponses followStats;
+        private final AutoFollowStats autoFollowStats;
+        private final FollowStatsAction.StatsResponses followStats;
 
         public Response(AutoFollowStats autoFollowStats, FollowStatsAction.StatsResponses followStats) {
             this.autoFollowStats = Objects.requireNonNull(autoFollowStats);
             this.followStats = Objects.requireNonNull(followStats);
         }
 
-        public Response() {
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            autoFollowStats = new AutoFollowStats(in);
+            followStats = new FollowStatsAction.StatsResponses();
+            followStats.readFrom(in);
         }
 
         public AutoFollowStats getAutoFollowStats() {
@@ -77,14 +86,6 @@ public FollowStatsAction.StatsResponses getFollowStats() {
             return followStats;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            autoFollowStats = new AutoFollowStats(in);
-            followStats = new FollowStatsAction.StatsResponses();
-            followStats.readFrom(in);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
index 5ca0c30b031d8..43dd236c160fb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
@@ -43,6 +43,13 @@ public static class Request extends AcknowledgedRequest<Request> {
 
         private String name;
 
+        public Request() {
+        }
+
+        public Request(String name) {
+            this.name = name;
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -60,9 +67,8 @@ public void setName(String name) {
             this.name = name;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Request(StreamInput in) throws IOException {
+            super(in);
             name = in.readString();
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
index e18c5a8896591..72618926a8940 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
@@ -34,7 +35,12 @@ private GetAutoFollowPatternAction() {
 
     @Override
     public Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     @Override
@@ -62,9 +68,8 @@ public void setName(String name) {
             this.name = name;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Request(StreamInput in) throws IOException {
+            super(in);
             this.name = in.readOptionalString();
         }
 
@@ -90,21 +95,17 @@ public int hashCode() {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private Map<String, AutoFollowPattern> autoFollowPatterns;
+        private final Map<String, AutoFollowPattern> autoFollowPatterns;
 
         public Response(Map<String, AutoFollowPattern> autoFollowPatterns) {
             this.autoFollowPatterns = autoFollowPatterns;
         }
 
-        public Response() {
-        }
-
         public Map<String, AutoFollowPattern> getAutoFollowPatterns() {
             return autoFollowPatterns;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
+        public Response(StreamInput in) throws IOException {
             super.readFrom(in);
             autoFollowPatterns = in.readMap(StreamInput::readString, AutoFollowPattern::new);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
index 4076d340fe13e..6bf958bcc9edd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
@@ -43,6 +43,11 @@ public Request(String followIndex) {
         public Request() {
         }
 
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            followIndex = in.readString();
+        }
+
         public String getFollowIndex() {
             return followIndex;
         }
@@ -52,12 +57,6 @@ public ActionRequestValidationException validate() {
             return null;
         }
 
-        @Override
-        public void readFrom(final StreamInput in) throws IOException {
-            super.readFrom(in);
-            followIndex = in.readString();
-        }
-
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
index 51cb5917be52c..ac2e008d0b267 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
@@ -118,6 +118,9 @@ public static Request fromXContent(XContentParser parser, String name) throws IO
         private TimeValue maxRetryDelay;
         private TimeValue readPollTimeout;
 
+        public Request() {
+        }
+
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
@@ -260,9 +263,8 @@ public void setReadPollTimeout(TimeValue readPollTimeout) {
             this.readPollTimeout = readPollTimeout;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Request(StreamInput in) throws IOException {
+            super(in);
             name = in.readString();
             remoteCluster = in.readString();
             leaderIndexPatterns = in.readList(StreamInput::readString);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
index 21dc9ede4d05b..856f3e29f840d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -54,7 +55,12 @@ private PutFollowAction() {
 
     @Override
     public Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements IndicesRequest, ToXContentObject {
@@ -171,13 +177,11 @@ public IndicesOptions indicesOptions() {
             return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Request(StreamInput in) throws IOException {
+            super(in);
             remoteCluster = in.readString();
             leaderIndex = in.readString();
-            followRequest = new ResumeFollowAction.Request();
-            followRequest.readFrom(in);
+            followRequest = new ResumeFollowAction.Request(in);
         }
 
         @Override
@@ -218,13 +222,9 @@ public int hashCode() {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private boolean followIndexCreated;
-        private boolean followIndexShardsAcked;
-        private boolean indexFollowingStarted;
-
-        public Response() {
-
-        }
+        private final boolean followIndexCreated;
+        private final boolean followIndexShardsAcked;
+        private final boolean indexFollowingStarted;
 
         public Response(boolean followIndexCreated, boolean followIndexShardsAcked, boolean indexFollowingStarted) {
             this.followIndexCreated = followIndexCreated;
@@ -244,9 +244,8 @@ public boolean isIndexFollowingStarted() {
             return indexFollowingStarted;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Response(StreamInput in) throws IOException {
+            super(in);
             followIndexCreated = in.readBoolean();
             followIndexShardsAcked = in.readBoolean();
             indexFollowingStarted = in.readBoolean();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
index 5125b4e1fb971..7439d521e21f8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
@@ -7,11 +7,11 @@
 package org.elasticsearch.xpack.core.ccr.action;
 
 import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -46,7 +46,7 @@ public AcknowledgedResponse newResponse() {
         return new AcknowledgedResponse();
     }
 
-    public static class Request extends ActionRequest implements ToXContentObject {
+    public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
 
         static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index");
         static final ParseField MAX_READ_REQUEST_OPERATION_COUNT = new ParseField("max_read_request_operation_count");
@@ -264,9 +264,8 @@ public ActionRequestValidationException validate() {
             return e;
         }
 
-        @Override
-        public void readFrom(final StreamInput in) throws IOException {
-            super.readFrom(in);
+        public Request(StreamInput in) throws IOException {
+            super(in);
             followerIndex = in.readString();
             maxReadRequestOperationCount = in.readOptionalVInt();
             maxOutstandingReadRequests = in.readOptionalVInt();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
index ab862ac9a101a..769c350cce6e9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
@@ -48,6 +48,11 @@ public Request(String followerIndex) {
             this.followerIndex = followerIndex;
         }
 
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            followerIndex = in.readString();
+        }
+
         public Request() {
         }
 
@@ -78,12 +83,6 @@ public ActionRequestValidationException validate() {
             return e;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            followerIndex = in.readString();
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateAction.java
new file mode 100644
index 0000000000000..7843fa7d86e0c
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateAction.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class AllocateAction implements LifecycleAction {
+
+    public static final String NAME = "allocate";
+    public static final ParseField NUMBER_OF_REPLICAS_FIELD = new ParseField("number_of_replicas");
+    public static final ParseField INCLUDE_FIELD = new ParseField("include");
+    public static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
+    public static final ParseField REQUIRE_FIELD = new ParseField("require");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+            a -> new AllocateAction((Integer) a[0], (Map<String, String>) a[1], (Map<String, String>) a[2], (Map<String, String>) a[3]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUMBER_OF_REPLICAS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), INCLUDE_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), EXCLUDE_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapStrings(), REQUIRE_FIELD);
+    }
+
+    private final Integer numberOfReplicas;
+    private final Map<String, String> include;
+    private final Map<String, String> exclude;
+    private final Map<String, String> require;
+
+    public static AllocateAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public AllocateAction(Integer numberOfReplicas, Map<String, String> include, Map<String, String> exclude, Map<String, String> require) {
+        if (include == null) {
+            this.include = Collections.emptyMap();
+        } else {
+            this.include = include;
+        }
+        if (exclude == null) {
+            this.exclude = Collections.emptyMap();
+        } else {
+            this.exclude = exclude;
+        }
+        if (require == null) {
+            this.require = Collections.emptyMap();
+        } else {
+            this.require = require;
+        }
+        if (this.include.isEmpty() && this.exclude.isEmpty() && this.require.isEmpty() && numberOfReplicas == null) {
+            throw new IllegalArgumentException(
+                    "At least one of " + INCLUDE_FIELD.getPreferredName() + ", " + EXCLUDE_FIELD.getPreferredName() + " or "
+                            + REQUIRE_FIELD.getPreferredName() + "must contain attributes for action " + NAME);
+        }
+        if (numberOfReplicas != null && numberOfReplicas < 0) {
+            throw new IllegalArgumentException("[" + NUMBER_OF_REPLICAS_FIELD.getPreferredName() + "] must be >= 0");
+        }
+        this.numberOfReplicas = numberOfReplicas;
+    }
+
+    @SuppressWarnings("unchecked")
+    public AllocateAction(StreamInput in) throws IOException {
+        this(in.readOptionalVInt(), (Map<String, String>) in.readGenericValue(), (Map<String, String>) in.readGenericValue(),
+                (Map<String, String>) in.readGenericValue());
+    }
+
+    public Integer getNumberOfReplicas() {
+        return numberOfReplicas;
+    }
+
+    public Map<String, String> getInclude() {
+        return include;
+    }
+
+    public Map<String, String> getExclude() {
+        return exclude;
+    }
+
+    public Map<String, String> getRequire() {
+        return require;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalVInt(numberOfReplicas);
+        out.writeGenericValue(include);
+        out.writeGenericValue(exclude);
+        out.writeGenericValue(require);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (numberOfReplicas != null) {
+            builder.field(NUMBER_OF_REPLICAS_FIELD.getPreferredName(), numberOfReplicas);
+        }
+        builder.field(INCLUDE_FIELD.getPreferredName(), include);
+        builder.field(EXCLUDE_FIELD.getPreferredName(), exclude);
+        builder.field(REQUIRE_FIELD.getPreferredName(), require);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return true;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
+        StepKey allocateKey = new StepKey(phase, NAME, NAME);
+        StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
+
+        Settings.Builder newSettings = Settings.builder();
+        if (numberOfReplicas != null) {
+            newSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas);
+        }
+        include.forEach((key, value) -> newSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + key, value));
+        exclude.forEach((key, value) -> newSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + key, value));
+        require.forEach((key, value) -> newSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + key, value));
+        UpdateSettingsStep allocateStep = new UpdateSettingsStep(allocateKey, allocationRoutedKey, client, newSettings.build());
+        AllocationRoutedStep routedCheckStep = new AllocationRoutedStep(allocationRoutedKey, nextStepKey);
+        return Arrays.asList(allocateStep, routedCheckStep);
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        StepKey allocateKey = new StepKey(phase, NAME, NAME);
+        StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
+        return Arrays.asList(allocateKey, allocationRoutedKey);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(numberOfReplicas, include, exclude, require);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        AllocateAction other = (AllocateAction) obj;
+        return Objects.equals(numberOfReplicas, other.numberOfReplicas) &&
+            Objects.equals(include, other.include) &&
+            Objects.equals(exclude, other.exclude) &&
+            Objects.equals(require, other.require);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStep.java
new file mode 100644
index 0000000000000..b97f7168a7f2b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStep.java
@@ -0,0 +1,185 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.ImmutableOpenIntMap;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Objects;
+
+public class AllocationRoutedStep extends ClusterStateWaitStep {
+    public static final String NAME = "check-allocation";
+
+    private static final Logger logger = LogManager.getLogger(AllocationRoutedStep.class);
+
+    private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(Collections.singletonList(
+            new FilterAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
+
+    AllocationRoutedStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public Result isConditionMet(Index index, ClusterState clusterState) {
+        IndexMetaData idxMeta = clusterState.metaData().index(index);
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            return new Result(false, null);
+        }
+        if (ActiveShardCount.ALL.enoughShardsActive(clusterState, index.getName()) == false) {
+            logger.debug("[{}] lifecycle action for index [{}] cannot make progress because not all shards are active",
+                    getKey().getAction(), index.getName());
+            return new Result(false, new Info(idxMeta.getNumberOfReplicas(), -1, false));
+        }
+        // All the allocation attributes are already set so just need to check
+        // if the allocation has happened
+        RoutingAllocation allocation = new RoutingAllocation(ALLOCATION_DECIDERS, clusterState.getRoutingNodes(), clusterState, null,
+                System.nanoTime());
+
+        int allocationPendingAllShards = 0;
+
+        ImmutableOpenIntMap<IndexShardRoutingTable> allShards = clusterState.getRoutingTable().index(index).getShards();
+        for (ObjectCursor<IndexShardRoutingTable> shardRoutingTable : allShards.values()) {
+            for (ShardRouting shardRouting : shardRoutingTable.value.shards()) {
+                String currentNodeId = shardRouting.currentNodeId();
+                boolean canRemainOnCurrentNode = ALLOCATION_DECIDERS
+                        .canRemain(shardRouting, clusterState.getRoutingNodes().node(currentNodeId), allocation)
+                        .type() == Decision.Type.YES;
+                if (canRemainOnCurrentNode == false) {
+                    allocationPendingAllShards++;
+                }
+            }
+        }
+
+        if (allocationPendingAllShards > 0) {
+            logger.debug("{} lifecycle action [{}] waiting for [{}] shards to be allocated to nodes matching the given filters",
+                index, getKey().getAction(), allocationPendingAllShards);
+            return new Result(false, new Info(idxMeta.getNumberOfReplicas(), allocationPendingAllShards, true));
+        } else {
+            logger.debug("{} lifecycle action for [{}] complete", index, getKey().getAction());
+            return new Result(true, null);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return 611;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        return super.equals(obj);
+    }
+
+    public static final class Info implements ToXContentObject {
+
+        private final long actualReplicas;
+        private final long numberShardsLeftToAllocate;
+        private final boolean allShardsActive;
+        private final String message;
+
+        static final ParseField ACTUAL_REPLICAS = new ParseField("actual_replicas");
+        static final ParseField SHARDS_TO_ALLOCATE = new ParseField("shards_left_to_allocate");
+        static final ParseField ALL_SHARDS_ACTIVE = new ParseField("all_shards_active");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<Info, Void> PARSER = new ConstructingObjectParser<>("allocation_routed_step_info",
+                a -> new Info((long) a[0], (long) a[1], (boolean) a[2]));
+        static {
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), ACTUAL_REPLICAS);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_ALLOCATE);
+            PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ALL_SHARDS_ACTIVE);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(long actualReplicas, long numberShardsLeftToAllocate, boolean allShardsActive) {
+            this.actualReplicas = actualReplicas;
+            this.numberShardsLeftToAllocate = numberShardsLeftToAllocate;
+            this.allShardsActive = allShardsActive;
+            if (allShardsActive == false) {
+                message = "Waiting for all shard copies to be active";
+            } else {
+                message = "Waiting for [" + numberShardsLeftToAllocate + "] shards "
+                        + "to be allocated to nodes matching the given filters";
+            }
+        }
+
+        public long getActualReplicas() {
+            return actualReplicas;
+        }
+
+        public long getNumberShardsLeftToAllocate() {
+            return numberShardsLeftToAllocate;
+        }
+
+        public boolean allShardsActive() {
+            return allShardsActive;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(), message);
+            builder.field(SHARDS_TO_ALLOCATE.getPreferredName(), numberShardsLeftToAllocate);
+            builder.field(ALL_SHARDS_ACTIVE.getPreferredName(), allShardsActive);
+            builder.field(ACTUAL_REPLICAS.getPreferredName(), actualReplicas);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(actualReplicas, numberShardsLeftToAllocate, allShardsActive);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Info other = (Info) obj;
+            return Objects.equals(actualReplicas, other.actualReplicas) &&
+                Objects.equals(numberShardsLeftToAllocate, other.numberShardsLeftToAllocate) &&
+                Objects.equals(allShardsActive, other.allShardsActive);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncActionStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncActionStep.java
new file mode 100644
index 0000000000000..4e35ef60a09d6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncActionStep.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+public abstract class AsyncActionStep extends Step {
+
+    private Client client;
+
+    public AsyncActionStep(StepKey key, StepKey nextStepKey, Client client) {
+        super(key, nextStepKey);
+        this.client = client;
+    }
+
+    protected Client getClient() {
+        return client;
+    }
+
+    public boolean indexSurvives() {
+        return true;
+    }
+
+    public abstract void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState, Listener listener);
+
+    public interface Listener {
+
+        void onResponse(boolean complete);
+
+        void onFailure(Exception e);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncWaitStep.java
new file mode 100644
index 0000000000000..f6c968cfae41a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/AsyncWaitStep.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+public abstract class AsyncWaitStep extends Step {
+
+    private Client client;
+
+    public AsyncWaitStep(StepKey key, StepKey nextStepKey, Client client) {
+        super(key, nextStepKey);
+        this.client = client;
+    }
+
+    protected Client getClient() {
+        return client;
+    }
+
+    public abstract void evaluateCondition(IndexMetaData indexMetaData, Listener listener);
+
+    public interface Listener {
+
+        void onResponse(boolean conditionMet, ToXContentObject infomationContext);
+
+        void onFailure(Exception e);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStep.java
new file mode 100644
index 0000000000000..bf29a595c192c
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStep.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * This step is used prior to running a shrink step in order to ensure that the index being shrunk
+ * has a copy of each shard allocated on one particular node (the node used by the require
+ * parameter) and that the shards are not relocating.
+ */
+public class CheckShrinkReadyStep extends ClusterStateWaitStep {
+    public static final String NAME = "check-shrink-allocation";
+
+    private static final Logger logger = LogManager.getLogger(CheckShrinkReadyStep.class);
+
+    CheckShrinkReadyStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public Result isConditionMet(Index index, ClusterState clusterState) {
+        IndexMetaData idxMeta = clusterState.metaData().index(index);
+
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists",
+                getKey().getAction(), index.getName());
+            return new Result(false, null);
+        }
+
+        // How many shards the node should have
+        int expectedShardCount = idxMeta.getNumberOfShards();
+
+        if (ActiveShardCount.ALL.enoughShardsActive(clusterState, index.getName()) == false) {
+            logger.debug("[{}] shrink action for [{}] cannot make progress because not all shards are active",
+                getKey().getAction(), index.getName());
+            return new Result(false, new CheckShrinkReadyStep.Info("", expectedShardCount, -1));
+        }
+
+        // The id of the node the shards should be on
+        final String idShardsShouldBeOn = idxMeta.getSettings().get(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id");
+        if (idShardsShouldBeOn == null) {
+            throw new IllegalStateException("Cannot check shrink allocation as there are no allocation rules by _id");
+        }
+
+        final IndexRoutingTable routingTable = clusterState.getRoutingTable().index(index);
+        int foundShards = 0;
+        for (ShardRouting shard : routingTable.shardsWithState(ShardRoutingState.STARTED)) {
+            final String currentNodeId = shard.currentNodeId();
+            if (idShardsShouldBeOn.equals(currentNodeId) && shard.relocating() == false) {
+                foundShards++;
+            }
+        }
+
+        logger.trace("{} checking for shrink readiness on [{}], found {} shards and need {}",
+            index, idShardsShouldBeOn, foundShards, expectedShardCount);
+
+        if (foundShards == expectedShardCount) {
+            logger.trace("{} successfully found {} allocated shards for shrink readiness on node [{}] ({})",
+                index, expectedShardCount, idShardsShouldBeOn, getKey().getAction());
+            return new Result(true, null);
+        } else {
+            logger.trace("{} failed to find {} allocated shards (found {}) on node [{}] for shrink readiness ({})",
+                index, expectedShardCount, foundShards, idShardsShouldBeOn, getKey().getAction());
+            return new Result(false, new CheckShrinkReadyStep.Info(idShardsShouldBeOn, expectedShardCount,
+                expectedShardCount - foundShards));
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return 612;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        return super.equals(obj);
+    }
+
+    public static final class Info implements ToXContentObject {
+
+        private final String nodeId;
+        private final long actualReplicas;
+        private final long numberShardsLeftToAllocate;
+        private final String message;
+
+        static final ParseField NODE_ID = new ParseField("node_id");
+        static final ParseField EXPECTED_SHARDS = new ParseField("expected_shards");
+        static final ParseField SHARDS_TO_ALLOCATE = new ParseField("shards_left_to_allocate");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<CheckShrinkReadyStep.Info, Void> PARSER = new ConstructingObjectParser<>(
+            "check_shrink_ready_step_info", a -> new CheckShrinkReadyStep.Info((String) a[0], (long) a[1], (long) a[2]));
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), EXPECTED_SHARDS);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_ALLOCATE);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(String nodeId, long expectedShards, long numberShardsLeftToAllocate) {
+            this.nodeId = nodeId;
+            this.actualReplicas = expectedShards;
+            this.numberShardsLeftToAllocate = numberShardsLeftToAllocate;
+            if (numberShardsLeftToAllocate < 0) {
+                this.message = "Waiting for all shards to become active";
+            } else {
+                this.message = String.format(Locale.ROOT, "Waiting for node [%s] to contain [%d] shards, found [%d], remaining [%d]",
+                    nodeId, expectedShards, expectedShards - numberShardsLeftToAllocate, numberShardsLeftToAllocate);
+            }
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(), message);
+            builder.field(NODE_ID.getPreferredName(), nodeId);
+            builder.field(SHARDS_TO_ALLOCATE.getPreferredName(), numberShardsLeftToAllocate);
+            builder.field(EXPECTED_SHARDS.getPreferredName(), actualReplicas);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(nodeId, actualReplicas, numberShardsLeftToAllocate);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            CheckShrinkReadyStep.Info other = (CheckShrinkReadyStep.Info) obj;
+            return Objects.equals(actualReplicas, other.actualReplicas) &&
+                Objects.equals(numberShardsLeftToAllocate, other.numberShardsLeftToAllocate) &&
+                Objects.equals(nodeId, other.nodeId);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateActionStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateActionStep.java
new file mode 100644
index 0000000000000..ae64de497886a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateActionStep.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.index.Index;
+
+public abstract class ClusterStateActionStep extends Step {
+
+    public ClusterStateActionStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    public abstract ClusterState performAction(Index index, ClusterState clusterState);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateWaitStep.java
new file mode 100644
index 0000000000000..0468f75490d9e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ClusterStateWaitStep.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.index.Index;
+
+public abstract class ClusterStateWaitStep extends Step {
+
+    public ClusterStateWaitStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    public abstract Result isConditionMet(Index index, ClusterState clusterState);
+
+    public static class Result {
+        private final boolean complete;
+        private final ToXContentObject infomationContext;
+
+        public Result(boolean complete, ToXContentObject infomationContext) {
+            this.complete = complete;
+            this.infomationContext = infomationContext;
+        }
+
+        public boolean isComplete() {
+            return complete;
+        }
+
+        public ToXContentObject getInfomationContext() {
+            return infomationContext;
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStep.java
new file mode 100644
index 0000000000000..b8192dd7e43be
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStep.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.index.Index;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
+/**
+ * Copies the execution state data from one index to another, typically after a
+ * new index has been created. Useful for actions such as shrink.
+ */
+public class CopyExecutionStateStep extends ClusterStateActionStep {
+    public static final String NAME = "copy_execution_state";
+
+    private static final Logger logger = LogManager.getLogger(CopyExecutionStateStep.class);
+    
+    private String shrunkIndexPrefix;
+
+
+    public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
+        super(key, nextStepKey);
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public ClusterState performAction(Index index, ClusterState clusterState) {
+        IndexMetaData indexMetaData = clusterState.metaData().index(index);
+        if (indexMetaData == null) {
+            // Index must have been since deleted, ignore it
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            return clusterState;
+        }
+        // get source index
+        String indexName = indexMetaData.getIndex().getName();
+        // get target shrink index
+        String targetIndexName = shrunkIndexPrefix + indexName;
+        IndexMetaData targetIndexMetaData = clusterState.metaData().index(targetIndexName);
+
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        String phase = lifecycleState.getPhase();
+        String action = lifecycleState.getAction();
+        long lifecycleDate = lifecycleState.getLifecycleDate();
+
+        LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder();
+        relevantTargetCustomData.setIndexCreationDate(lifecycleDate);
+        relevantTargetCustomData.setPhase(phase);
+        relevantTargetCustomData.setAction(action);
+        relevantTargetCustomData.setStep(ShrunkenIndexCheckStep.NAME);
+
+        MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
+            .put(IndexMetaData.builder(targetIndexMetaData)
+                .putCustom(ILM_CUSTOM_METADATA_KEY, relevantTargetCustomData.build().asMap()));
+
+        return ClusterState.builder(clusterState).metaData(newMetaData).build();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        CopyExecutionStateStep that = (CopyExecutionStateStep) o;
+        return Objects.equals(shrunkIndexPrefix, that.shrunkIndexPrefix);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteAction.java
new file mode 100644
index 0000000000000..1a0ad4c789ce4
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteAction.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A {@link LifecycleAction} which deletes the index.
+ */
+public class DeleteAction implements LifecycleAction {
+    public static final String NAME = "delete";
+
+    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, DeleteAction::new);
+
+    public static DeleteAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public DeleteAction() {
+    }
+
+    public DeleteAction(StreamInput in) throws IOException {
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return true;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        Step.StepKey deleteStepKey = new Step.StepKey(phase, NAME, DeleteStep.NAME);
+        return Collections.singletonList(new DeleteStep(deleteStepKey, nextStepKey, client));
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        return Collections.singletonList(new Step.StepKey(phase, NAME, DeleteStep.NAME));
+    }
+
+    @Override
+    public int hashCode() {
+        return 1;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStep.java
new file mode 100644
index 0000000000000..b5ae441388419
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStep.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+public class DeleteStep extends AsyncActionStep {
+    public static final String NAME = "delete";
+
+    public DeleteStep(StepKey key, StepKey nextStepKey, Client client) {
+        super(key, nextStepKey, client);
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+        getClient().admin().indices()
+            .delete(new DeleteIndexRequest(indexMetaData.getIndex().getName()),
+                ActionListener.wrap(response -> listener.onResponse(true) , listener::onFailure));
+    }
+
+    @Override
+    public boolean indexSurvives() {
+        return false;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStep.java
new file mode 100644
index 0000000000000..50ad0155dff29
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStep.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+public class ErrorStep extends Step {
+    public static final String NAME = "ERROR";
+
+    public ErrorStep(StepKey key) {
+        super(key, key);
+        if (NAME.equals(key.getName()) == false) {
+            throw new IllegalArgumentException("An error step must have a step key whose step name is " + NAME);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequest.java
new file mode 100644
index 0000000000000..037de2d505292
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.info.ClusterInfoRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * The request object used by the Explain Lifecycle API.
+ * 
+ * Multiple indices may be queried in the same request using the
+ * {@link #indices(String...)} method
+ */
+public class ExplainLifecycleRequest extends ClusterInfoRequest<ExplainLifecycleRequest> {
+
+    public ExplainLifecycleRequest() {
+        super();
+    }
+
+    public ExplainLifecycleRequest(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.hashCode(indices()), indicesOptions());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ExplainLifecycleRequest other = (ExplainLifecycleRequest) obj;
+        return Objects.deepEquals(indices(), other.indices()) &&
+                Objects.equals(indicesOptions(), other.indicesOptions());
+    }
+
+    @Override
+    public String toString() {
+        return "ExplainLifecycleRequest [indices()=" + Arrays.toString(indices()) + ", indicesOptions()=" + indicesOptions() + "]";
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponse.java
new file mode 100644
index 0000000000000..915ca17cb43a4
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponse.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * The response object returned by the Explain Lifecycle API.
+ * 
+ * Since the API can be run over multiple indices the response provides a map of
+ * index to the explanation of the lifecycle status for that index.
+ */
+public class ExplainLifecycleResponse extends ActionResponse implements ToXContentObject {
+
+    public static final ParseField INDICES_FIELD = new ParseField("indices");
+
+    private Map<String, IndexLifecycleExplainResponse> indexResponses;
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ExplainLifecycleResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "explain_lifecycle_response", a -> new ExplainLifecycleResponse(((List<IndexLifecycleExplainResponse>) a[0]).stream()
+                    .collect(Collectors.toMap(IndexLifecycleExplainResponse::getIndex, Function.identity()))));
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> IndexLifecycleExplainResponse.PARSER.apply(p, c),
+                INDICES_FIELD);
+    }
+
+    public static ExplainLifecycleResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ExplainLifecycleResponse() {
+    }
+
+    public ExplainLifecycleResponse(Map<String, IndexLifecycleExplainResponse> indexResponses) {
+        this.indexResponses = indexResponses;
+    }
+
+    /**
+     * @return a map of the responses from each requested index. The maps key is
+     *         the index name and the value is the
+     *         {@link IndexLifecycleExplainResponse} describing the current
+     *         lifecycle status of that index
+     */
+    public Map<String, IndexLifecycleExplainResponse> getIndexResponses() {
+        return indexResponses;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startObject(INDICES_FIELD.getPreferredName());
+        for (IndexLifecycleExplainResponse indexResponse : indexResponses.values()) {
+            builder.field(indexResponse.getIndex(), indexResponse);
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            IndexLifecycleExplainResponse indexResponse = new IndexLifecycleExplainResponse(in);
+            indexResponses.put(indexResponse.getIndex(), indexResponse);
+        }
+        this.indexResponses = indexResponses;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(indexResponses.size());
+        for (IndexLifecycleExplainResponse e : indexResponses.values()) {
+            e.writeTo(out);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indexResponses);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ExplainLifecycleResponse other = (ExplainLifecycleResponse) obj;
+        return Objects.equals(indexResponses, other.indexResponses);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeAction.java
new file mode 100644
index 0000000000000..2c4508a8355f0
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeAction.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A {@link LifecycleAction} which force-merges the index.
+ */
+public class ForceMergeAction implements LifecycleAction {
+    public static final String NAME = "forcemerge";
+    public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
+
+    private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+        false, a -> {
+        int maxNumSegments = (int) a[0];
+        return new ForceMergeAction(maxNumSegments);
+    });
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), MAX_NUM_SEGMENTS_FIELD);
+    }
+
+    private final int maxNumSegments;
+
+    public static ForceMergeAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ForceMergeAction(int maxNumSegments) {
+        if (maxNumSegments <= 0) {
+            throw new IllegalArgumentException("[" + MAX_NUM_SEGMENTS_FIELD.getPreferredName()
+                + "] must be a positive integer");
+        }
+        this.maxNumSegments = maxNumSegments;
+    }
+
+    public ForceMergeAction(StreamInput in) throws IOException {
+        this.maxNumSegments = in.readVInt();
+    }
+
+    public int getMaxNumSegments() {
+        return maxNumSegments;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(maxNumSegments);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return true;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(MAX_NUM_SEGMENTS_FIELD.getPreferredName(), maxNumSegments);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
+
+        StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
+        StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME);
+        StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME);
+
+        UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, forceMergeKey, client, readOnlySettings);
+        ForceMergeStep forceMergeStep = new ForceMergeStep(forceMergeKey, countKey, client, maxNumSegments);
+        SegmentCountStep segmentCountStep = new SegmentCountStep(countKey, nextStepKey, client, maxNumSegments);
+        return Arrays.asList(readOnlyStep, forceMergeStep, segmentCountStep);
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
+        StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME);
+        StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME);
+        return Arrays.asList(readOnlyKey, forceMergeKey, countKey);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxNumSegments);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        ForceMergeAction other = (ForceMergeAction) obj;
+        return Objects.equals(maxNumSegments, other.maxNumSegments);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStep.java
new file mode 100644
index 0000000000000..776043babf0fc
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStep.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+import java.util.Objects;
+
+public class ForceMergeStep extends AsyncActionStep {
+    public static final String NAME = "forcemerge";
+    private final int maxNumSegments;
+
+    public ForceMergeStep(StepKey key, StepKey nextStepKey, Client client, int maxNumSegments) {
+        super(key, nextStepKey, client);
+        this.maxNumSegments = maxNumSegments;
+    }
+
+    public int getMaxNumSegments() {
+        return maxNumSegments;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+        ForceMergeRequest request = new ForceMergeRequest(indexMetaData.getIndex().getName());
+        request.maxNumSegments(maxNumSegments);
+        getClient().admin().indices()
+            .forceMerge(request, ActionListener.wrap(response -> listener.onResponse(true),
+                listener::onFailure));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), maxNumSegments);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ForceMergeStep other = (ForceMergeStep) obj;
+        return super.equals(obj) &&
+            Objects.equals(maxNumSegments, other.maxNumSegments);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java
new file mode 100644
index 0000000000000..7e97170998a81
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java
@@ -0,0 +1,308 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class IndexLifecycleExplainResponse implements ToXContentObject, Writeable {
+
+    private static final ParseField INDEX_FIELD = new ParseField("index");
+    private static final ParseField MANAGED_BY_ILM_FIELD = new ParseField("managed");
+    private static final ParseField POLICY_NAME_FIELD = new ParseField("policy");
+    private static final ParseField LIFECYCLE_DATE_MILLIS_FIELD = new ParseField("lifecycle_date_millis");
+    private static final ParseField LIFECYCLE_DATE_FIELD = new ParseField("lifecycle_date");
+    private static final ParseField PHASE_FIELD = new ParseField("phase");
+    private static final ParseField ACTION_FIELD = new ParseField("action");
+    private static final ParseField STEP_FIELD = new ParseField("step");
+    private static final ParseField FAILED_STEP_FIELD = new ParseField("failed_step");
+    private static final ParseField PHASE_TIME_MILLIS_FIELD = new ParseField("phase_time_millis");
+    private static final ParseField PHASE_TIME_FIELD = new ParseField("phase_time");
+    private static final ParseField ACTION_TIME_MILLIS_FIELD = new ParseField("action_time_millis");
+    private static final ParseField ACTION_TIME_FIELD = new ParseField("action_time");
+    private static final ParseField STEP_TIME_MILLIS_FIELD = new ParseField("step_time_millis");
+    private static final ParseField STEP_TIME_FIELD = new ParseField("step_time");
+    private static final ParseField STEP_INFO_FIELD = new ParseField("step_info");
+    private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution");
+
+    public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
+            "index_lifecycle_explain_response",
+            a -> new IndexLifecycleExplainResponse(
+                    (String) a[0],
+                    (boolean) a[1],
+                    (String) a[2],
+                    (Long) (a[3]),
+                    (String) a[4],
+                    (String) a[5],
+                    (String) a[6],
+                    (String) a[7],
+                    (Long) (a[8]),
+                    (Long) (a[9]),
+                    (Long) (a[10]),
+                    (BytesReference) a[11],
+                    (PhaseExecutionInfo) a[12]));
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), MANAGED_BY_ILM_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), POLICY_NAME_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), LIFECYCLE_DATE_MILLIS_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PHASE_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ACTION_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), STEP_FIELD);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FAILED_STEP_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), PHASE_TIME_MILLIS_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), ACTION_TIME_MILLIS_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), STEP_TIME_MILLIS_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.copyCurrentStructure(p);
+            return BytesArray.bytes(builder);
+        }, STEP_INFO_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> PhaseExecutionInfo.parse(p, ""),
+            PHASE_EXECUTION_INFO);
+    }
+
+    private final String index;
+    private final String policyName;
+    private final String phase;
+    private final String action;
+    private final String step;
+    private final String failedStep;
+    private final Long lifecycleDate;
+    private final Long phaseTime;
+    private final Long actionTime;
+    private final Long stepTime;
+    private final boolean managedByILM;
+    private final BytesReference stepInfo;
+    private final PhaseExecutionInfo phaseExecutionInfo;
+
+    public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, Long lifecycleDate,
+            String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime, Long stepTime,
+            BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
+        return new IndexLifecycleExplainResponse(index, true, policyName, lifecycleDate, phase, action, step, failedStep, phaseTime,
+                actionTime, stepTime, stepInfo, phaseExecutionInfo);
+    }
+
+    public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String index) {
+        return new IndexLifecycleExplainResponse(index, false, null, null, null, null, null, null, null, null, null, null, null);
+    }
+
+    private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, Long lifecycleDate,
+                                          String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime,
+                                          Long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
+        if (managedByILM) {
+            if (policyName == null) {
+                throw new IllegalArgumentException("[" + POLICY_NAME_FIELD.getPreferredName() + "] cannot be null for managed index");
+            }
+        } else {
+            if (policyName != null || lifecycleDate != null || phase != null || action != null || step != null || failedStep != null
+                    || phaseTime != null || actionTime != null || stepTime != null || stepInfo != null || phaseExecutionInfo != null) {
+                throw new IllegalArgumentException(
+                        "Unmanaged index response must only contain fields: [" + MANAGED_BY_ILM_FIELD + ", " + INDEX_FIELD + "]");
+            }
+        }
+        this.index = index;
+        this.policyName = policyName;
+        this.managedByILM = managedByILM;
+        this.lifecycleDate = lifecycleDate;
+        this.phase = phase;
+        this.action = action;
+        this.step = step;
+        this.phaseTime = phaseTime;
+        this.actionTime = actionTime;
+        this.stepTime = stepTime;
+        this.failedStep = failedStep;
+        this.stepInfo = stepInfo;
+        this.phaseExecutionInfo = phaseExecutionInfo;
+    }
+
+    public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
+        index = in.readString();
+        managedByILM = in.readBoolean();
+        if (managedByILM) {
+            policyName = in.readString();
+            lifecycleDate = in.readOptionalLong();
+            phase = in.readOptionalString();
+            action = in.readOptionalString();
+            step = in.readOptionalString();
+            failedStep = in.readOptionalString();
+            phaseTime = in.readOptionalLong();
+            actionTime = in.readOptionalLong();
+            stepTime = in.readOptionalLong();
+            stepInfo = in.readOptionalBytesReference();
+            phaseExecutionInfo = in.readOptionalWriteable(PhaseExecutionInfo::new);
+        } else {
+            policyName = null;
+            lifecycleDate = null;
+            phase = null;
+            action = null;
+            step = null;
+            failedStep = null;
+            phaseTime = null;
+            actionTime = null;
+            stepTime = null;
+            stepInfo = null;
+            phaseExecutionInfo = null;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(index);
+        out.writeBoolean(managedByILM);
+        if (managedByILM) {
+            out.writeString(policyName);
+            out.writeOptionalLong(lifecycleDate);
+            out.writeOptionalString(phase);
+            out.writeOptionalString(action);
+            out.writeOptionalString(step);
+            out.writeOptionalString(failedStep);
+            out.writeOptionalLong(phaseTime);
+            out.writeOptionalLong(actionTime);
+            out.writeOptionalLong(stepTime);
+            out.writeOptionalBytesReference(stepInfo);
+            out.writeOptionalWriteable(phaseExecutionInfo);
+        }
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public boolean managedByILM() {
+        return managedByILM;
+    }
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    public Long getLifecycleDate() {
+        return lifecycleDate;
+    }
+
+    public String getPhase() {
+        return phase;
+    }
+
+    public Long getPhaseTime() {
+        return phaseTime;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public Long getActionTime() {
+        return actionTime;
+    }
+
+    public String getStep() {
+        return step;
+    }
+
+    public Long getStepTime() {
+        return stepTime;
+    }
+
+    public String getFailedStep() {
+        return failedStep;
+    }
+
+    public BytesReference getStepInfo() {
+        return stepInfo;
+    }
+
+    public PhaseExecutionInfo getPhaseExecutionInfo() {
+        return phaseExecutionInfo;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX_FIELD.getPreferredName(), index);
+        builder.field(MANAGED_BY_ILM_FIELD.getPreferredName(), managedByILM);
+        if (managedByILM) {
+            builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
+            if (lifecycleDate != null) {
+                builder.timeField(LIFECYCLE_DATE_MILLIS_FIELD.getPreferredName(), LIFECYCLE_DATE_FIELD.getPreferredName(), lifecycleDate);
+            }
+            builder.field(PHASE_FIELD.getPreferredName(), phase);
+            if (phaseTime != null) {
+                builder.timeField(PHASE_TIME_MILLIS_FIELD.getPreferredName(), PHASE_TIME_FIELD.getPreferredName(), phaseTime);
+            }
+            builder.field(ACTION_FIELD.getPreferredName(), action);
+            if (actionTime != null) {
+                builder.timeField(ACTION_TIME_MILLIS_FIELD.getPreferredName(), ACTION_TIME_FIELD.getPreferredName(), actionTime);
+            }
+            builder.field(STEP_FIELD.getPreferredName(), step);
+            if (stepTime != null) {
+                builder.timeField(STEP_TIME_MILLIS_FIELD.getPreferredName(), STEP_TIME_FIELD.getPreferredName(), stepTime);
+            }
+            if (Strings.hasLength(failedStep)) {
+                builder.field(FAILED_STEP_FIELD.getPreferredName(), failedStep);
+            }
+            if (stepInfo != null && stepInfo.length() > 0) {
+                builder.rawField(STEP_INFO_FIELD.getPreferredName(), stepInfo.streamInput(), XContentType.JSON);
+            }
+            if (phaseExecutionInfo != null) {
+                builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo);
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, managedByILM, policyName, lifecycleDate, phase, action, step, failedStep, phaseTime, actionTime,
+                stepTime, stepInfo, phaseExecutionInfo);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        IndexLifecycleExplainResponse other = (IndexLifecycleExplainResponse) obj;
+        return Objects.equals(index, other.index) &&
+                Objects.equals(managedByILM, other.managedByILM) &&
+                Objects.equals(policyName, other.policyName) &&
+                Objects.equals(lifecycleDate, other.lifecycleDate) &&
+                Objects.equals(phase, other.phase) &&
+                Objects.equals(action, other.action) &&
+                Objects.equals(step, other.step) &&
+                Objects.equals(failedStep, other.failedStep) &&
+                Objects.equals(phaseTime, other.phaseTime) &&
+                Objects.equals(actionTime, other.actionTime) &&
+                Objects.equals(stepTime, other.stepTime) &&
+                Objects.equals(stepInfo, other.stepInfo) &&
+                Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsage.java
new file mode 100644
index 0000000000000..f0dd25eabac1f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsage.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class IndexLifecycleFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    private List<PolicyStats> policyStats;
+
+    public IndexLifecycleFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+        if (input.readBoolean()) {
+            policyStats = input.readList(PolicyStats::new);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        boolean hasPolicyStats = policyStats != null;
+        out.writeBoolean(hasPolicyStats);
+        if (hasPolicyStats) {
+            out.writeList(policyStats);
+        }
+    }
+
+    public IndexLifecycleFeatureSetUsage(boolean available, boolean enabled) {
+        this(available, enabled, null);
+    }
+
+    public IndexLifecycleFeatureSetUsage(boolean available, boolean enabled, List<PolicyStats> policyStats) {
+        super(XPackField.INDEX_LIFECYCLE, available, enabled);
+        this.policyStats = policyStats;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        if (policyStats != null) {
+            builder.field("policy_count", policyStats.size());
+            builder.field("policy_stats", policyStats);
+        }
+    }
+
+    public List<PolicyStats> getPolicyStats() {
+        return policyStats;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(available, enabled, policyStats);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        IndexLifecycleFeatureSetUsage other = (IndexLifecycleFeatureSetUsage) obj;
+        return Objects.equals(available, other.available) &&
+                Objects.equals(enabled, other.enabled) &&
+                Objects.equals(policyStats, other.policyStats);
+    }
+
+    public static final class PolicyStats implements ToXContentObject, Writeable {
+
+        public static final ParseField INDICES_MANAGED_FIELD = new ParseField("indices_managed");
+
+        private final Map<String, PhaseStats> phaseStats;
+        private final int indicesManaged;
+
+        public PolicyStats(Map<String, PhaseStats> phaseStats, int numberIndicesManaged) {
+            this.phaseStats = phaseStats;
+            this.indicesManaged = numberIndicesManaged;
+        }
+
+        public PolicyStats(StreamInput in) throws IOException {
+            this.phaseStats = in.readMap(StreamInput::readString, PhaseStats::new);
+            this.indicesManaged = in.readVInt();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeMap(phaseStats, StreamOutput::writeString, (o, p) -> p.writeTo(o));
+            out.writeVInt(indicesManaged);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(LifecyclePolicy.PHASES_FIELD.getPreferredName(), phaseStats);
+            builder.field(INDICES_MANAGED_FIELD.getPreferredName(), indicesManaged);
+            builder.endObject();
+            return builder;
+        }
+
+        public Map<String, PhaseStats> getPhaseStats() {
+            return phaseStats;
+        }
+
+        public int getIndicesManaged() {
+            return indicesManaged;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(phaseStats, indicesManaged);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            PolicyStats other = (PolicyStats) obj;
+            return Objects.equals(phaseStats, other.phaseStats) &&
+                    Objects.equals(indicesManaged, other.indicesManaged);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    public static final class PhaseStats implements ToXContentObject, Writeable {
+        private final String[] actionNames;
+        private final TimeValue minimumAge;
+
+        public PhaseStats(TimeValue after, String[] actionNames) {
+            this.actionNames = actionNames;
+            this.minimumAge = after;
+        }
+
+        public PhaseStats(StreamInput in) throws IOException {
+            actionNames = in.readStringArray();
+            minimumAge = in.readTimeValue();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeStringArray(actionNames);
+            out.writeTimeValue(minimumAge);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(Phase.MIN_AGE.getPreferredName(), minimumAge.getMillis());
+            builder.field(Phase.ACTIONS_FIELD.getPreferredName(), actionNames);
+            builder.endObject();
+            return builder;
+        }
+
+        public String[] getActionNames() {
+            return actionNames;
+        }
+
+        public TimeValue getAfter() {
+            return minimumAge;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(Arrays.hashCode(actionNames), minimumAge);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            PhaseStats other = (PhaseStats) obj;
+            return Objects.equals(minimumAge, other.minimumAge) &&
+                    Objects.deepEquals(actionNames, other.actionNames);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java
new file mode 100644
index 0000000000000..0d68f070b6ecb
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MetaData.Custom;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackPlugin.XPackMetaDataCustom;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+
+public class IndexLifecycleMetadata implements XPackMetaDataCustom {
+    public static final String TYPE = "index_lifecycle";
+    public static final ParseField OPERATION_MODE_FIELD = new ParseField("operation_mode");
+    public static final ParseField POLICIES_FIELD = new ParseField("policies");
+    public static final IndexLifecycleMetadata EMPTY = new IndexLifecycleMetadata(Collections.emptySortedMap(), OperationMode.RUNNING);
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<IndexLifecycleMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE,
+            a -> new IndexLifecycleMetadata(
+                    ((List<LifecyclePolicyMetadata>) a[0]).stream()
+                            .collect(Collectors.toMap(LifecyclePolicyMetadata::getName, Function.identity())),
+                    OperationMode.valueOf((String) a[1])));
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> LifecyclePolicyMetadata.parse(p, n),
+                v -> {
+                    throw new IllegalArgumentException("ordered " + POLICIES_FIELD.getPreferredName() + " are not supported");
+                }, POLICIES_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), OPERATION_MODE_FIELD);
+    }
+
+    private final Map<String, LifecyclePolicyMetadata> policyMetadatas;
+    private final OperationMode operationMode;
+
+    public IndexLifecycleMetadata(Map<String, LifecyclePolicyMetadata> policies, OperationMode operationMode) {
+        this.policyMetadatas = Collections.unmodifiableMap(policies);
+        this.operationMode = operationMode;
+    }
+
+    public IndexLifecycleMetadata(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        TreeMap<String, LifecyclePolicyMetadata> policies = new TreeMap<>();
+        for (int i = 0; i < size; i++) {
+            policies.put(in.readString(), new LifecyclePolicyMetadata(in));
+        }
+        this.policyMetadatas = policies;
+        this.operationMode = in.readEnum(OperationMode.class);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(policyMetadatas.size());
+        for (Map.Entry<String, LifecyclePolicyMetadata> entry : policyMetadatas.entrySet()) {
+            out.writeString(entry.getKey());
+            entry.getValue().writeTo(out);
+        }
+        out.writeEnum(operationMode);
+    }
+
+    public Map<String, LifecyclePolicyMetadata> getPolicyMetadatas() {
+        return policyMetadatas;
+    }
+
+    public OperationMode getOperationMode() {
+        return operationMode;
+    }
+
+    public Map<String, LifecyclePolicy> getPolicies() {
+        return policyMetadatas.values().stream().map(LifecyclePolicyMetadata::getPolicy)
+                .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity()));
+    }
+
+    @Override
+    public Diff<Custom> diff(Custom previousState) {
+        return new IndexLifecycleMetadataDiff((IndexLifecycleMetadata) previousState, this);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(POLICIES_FIELD.getPreferredName(), policyMetadatas);
+        builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode);
+        return builder;
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_6_5_0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(policyMetadatas, operationMode);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        IndexLifecycleMetadata other = (IndexLifecycleMetadata) obj;
+        return Objects.equals(policyMetadatas, other.policyMetadatas)
+            && Objects.equals(operationMode, other.operationMode);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    public static class IndexLifecycleMetadataDiff implements NamedDiff<MetaData.Custom> {
+
+        final Diff<Map<String, LifecyclePolicyMetadata>> policies;
+        final OperationMode operationMode;
+
+        IndexLifecycleMetadataDiff(IndexLifecycleMetadata before, IndexLifecycleMetadata after) {
+            this.policies = DiffableUtils.diff(before.policyMetadatas, after.policyMetadatas, DiffableUtils.getStringKeySerializer());
+            this.operationMode = after.operationMode;
+        }
+
+        public IndexLifecycleMetadataDiff(StreamInput in) throws IOException {
+            this.policies = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), LifecyclePolicyMetadata::new,
+                    IndexLifecycleMetadataDiff::readLifecyclePolicyDiffFrom);
+            this.operationMode = in.readEnum(OperationMode.class);
+        }
+
+        @Override
+        public MetaData.Custom apply(MetaData.Custom part) {
+            TreeMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(
+                    policies.apply(((IndexLifecycleMetadata) part).policyMetadatas));
+            return new IndexLifecycleMetadata(newPolicies, this.operationMode);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            policies.writeTo(out);
+            out.writeEnum(operationMode);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        static Diff<LifecyclePolicyMetadata> readLifecyclePolicyDiffFrom(StreamInput in) throws IOException {
+            return AbstractDiffable.readDiffFrom(LifecyclePolicyMetadata::new, in);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java
new file mode 100644
index 0000000000000..c9046cb5eb7ec
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.index.Index;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
+public final class InitializePolicyContextStep extends ClusterStateActionStep {
+    public static final String INITIALIZATION_PHASE = "new";
+    public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
+    private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
+
+    public InitializePolicyContextStep(Step.StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public ClusterState performAction(Index index, ClusterState clusterState) {
+        IndexMetaData indexMetaData = clusterState.getMetaData().index(index);
+        if (indexMetaData == null) {
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            // Index must have been since deleted, ignore it
+            return clusterState;
+        }
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(indexMetaData);
+        if (lifecycleState.getLifecycleDate() != null) {
+            return clusterState;
+        }
+
+        ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
+
+        LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
+        newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
+        newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
+            .builder(indexMetaData)
+            .putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
+        return newClusterStateBuilder.build();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleAction.java
new file mode 100644
index 0000000000000..3e84813274d83
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleAction.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.List;
+
+/**
+ * Executes an action on an index related to its lifecycle.
+ */
+public interface LifecycleAction extends ToXContentObject, NamedWriteable {
+
+    /**
+     * converts the {@link LifecycleAction}'s execution plan into a series of
+     * {@link Step}s that reference each other to preserve order of operations.
+     * @param client      the client that will be used by {@link AsyncActionStep} and {@link AsyncWaitStep} steps
+     * @param phase       the name of the phase this action is being executed within
+     * @param nextStepKey the next step to execute after this action's steps. If null, then there are no further
+     *                    steps to run. It is the responsibility of each {@link LifecycleAction} to implement this
+     *                    correctly and not forget to link to this final step so that the policy can continue.
+     * @return an ordered list of steps that represent the execution plan of the action
+     */
+    List<Step> toSteps(Client client, String phase, @Nullable Step.StepKey nextStepKey);
+
+    /**
+     * 
+     * @param phase
+     *            the name of the phase this action is being executed within
+     * @return the {@link StepKey}s for the steps which will be executed in this
+     *         action
+     */
+    List<StepKey> toStepKeys(String phase);
+
+    /**
+     * @return true if this action is considered safe. An action is not safe if
+     *         it will produce unwanted side effects or will get stuck when the
+     *         action configuration is changed while an index is in this action
+     */
+    boolean isSafeAction();
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionState.java
new file mode 100644
index 0000000000000..b2d42bca7338e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionState.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Contains information about the execution of a lifecycle policy for a single
+ * index, and serializes/deserializes this information to and from custom
+ * index metadata.
+ */
+public class LifecycleExecutionState {
+    public static final String ILM_CUSTOM_METADATA_KEY = "ilm";
+
+    private static final String PHASE = "phase";
+    private static final String ACTION = "action";
+    private static final String STEP = "step";
+    private static final String INDEX_CREATION_DATE = "creation_date";
+    private static final String PHASE_TIME = "phase_time";
+    private static final String ACTION_TIME = "action_time";
+    private static final String STEP_TIME = "step_time";
+    private static final String FAILED_STEP = "failed_step";
+    private static final String STEP_INFO = "step_info";
+    private static final String PHASE_DEFINITION = "phase_definition";
+
+    private final String phase;
+    private final String action;
+    private final String step;
+    private final String failedStep;
+    private final String stepInfo;
+    private final String phaseDefinition;
+    private final Long lifecycleDate;
+    private final Long phaseTime;
+    private final Long actionTime;
+    private final Long stepTime;
+
+    private LifecycleExecutionState(String phase, String action, String step, String failedStep,
+                                    String stepInfo, String phaseDefinition, Long lifecycleDate,
+                                    Long phaseTime, Long actionTime, Long stepTime) {
+        this.phase = phase;
+        this.action = action;
+        this.step = step;
+        this.failedStep = failedStep;
+        this.stepInfo = stepInfo;
+        this.phaseDefinition = phaseDefinition;
+        this.lifecycleDate = lifecycleDate;
+        this.phaseTime = phaseTime;
+        this.actionTime = actionTime;
+        this.stepTime = stepTime;
+    }
+
+    /**
+     * Retrieves the execution state from an {@link IndexMetaData} based on the
+     * custom metadata.
+     * @param indexMetaData The metadata of the index to retrieve the execution
+     *                      state from.
+     * @return The execution state of that index.
+     */
+    public static LifecycleExecutionState fromIndexMetadata(IndexMetaData indexMetaData) {
+        Map<String, String> customData = indexMetaData.getCustomData(ILM_CUSTOM_METADATA_KEY);
+        customData = customData == null ? new HashMap<>() : customData;
+        return fromCustomMetadata(customData);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static Builder builder(LifecycleExecutionState state) {
+        return new Builder()
+            .setPhase(state.phase)
+            .setAction(state.action)
+            .setStep(state.step)
+            .setFailedStep(state.failedStep)
+            .setStepInfo(state.stepInfo)
+            .setPhaseDefinition(state.phaseDefinition)
+            .setIndexCreationDate(state.lifecycleDate)
+            .setPhaseTime(state.phaseTime)
+            .setActionTime(state.actionTime)
+            .setStepTime(state.stepTime);
+    }
+
+    static LifecycleExecutionState fromCustomMetadata(Map<String, String> customData) {
+        Builder builder = builder();
+        if (customData.containsKey(PHASE)) {
+            builder.setPhase(customData.get(PHASE));
+        }
+        if (customData.containsKey(ACTION)) {
+            builder.setAction(customData.get(ACTION));
+        }
+        if (customData.containsKey(STEP)) {
+            builder.setStep(customData.get(STEP));
+        }
+        if (customData.containsKey(FAILED_STEP)) {
+            builder.setFailedStep(customData.get(FAILED_STEP));
+        }
+        if (customData.containsKey(STEP_INFO)) {
+            builder.setStepInfo(customData.get(STEP_INFO));
+        }
+        if (customData.containsKey(PHASE_DEFINITION)) {
+            builder.setPhaseDefinition(customData.get(PHASE_DEFINITION));
+        }
+        if (customData.containsKey(INDEX_CREATION_DATE)) {
+            try {
+                builder.setIndexCreationDate(Long.parseLong(customData.get(INDEX_CREATION_DATE)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, INDEX_CREATION_DATE, customData.get(INDEX_CREATION_DATE));
+            }
+        }
+        if (customData.containsKey(PHASE_TIME)) {
+            try {
+                builder.setPhaseTime(Long.parseLong(customData.get(PHASE_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, PHASE_TIME, customData.get(PHASE_TIME));
+            }
+        }
+        if (customData.containsKey(ACTION_TIME)) {
+            try {
+                builder.setActionTime(Long.parseLong(customData.get(ACTION_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, ACTION_TIME, customData.get(ACTION_TIME));
+            }
+        }
+        if (customData.containsKey(STEP_TIME)) {
+            try {
+                builder.setStepTime(Long.parseLong(customData.get(STEP_TIME)));
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchException("Custom metadata field [{}] does not contain a valid long. Actual value: [{}]",
+                    e, STEP_TIME, customData.get(STEP_TIME));
+            }
+        }
+        return builder.build();
+    }
+
+    /**
+     * Converts this object to an immutable map representation for use with
+     * {@link IndexMetaData.Builder#putCustom(String, Map)}.
+     * @return An immutable Map representation of this execution state.
+     */
+    public Map<String, String> asMap() {
+        Map<String, String> result = new HashMap<>();
+        if (phase != null) {
+            result.put(PHASE, phase);
+        }
+        if (action != null) {
+            result.put(ACTION, action);
+        }
+        if (step != null) {
+            result.put(STEP, step);
+        }
+        if (failedStep != null) {
+            result.put(FAILED_STEP, failedStep);
+        }
+        if (stepInfo != null) {
+            result.put(STEP_INFO, stepInfo);
+        }
+        if (lifecycleDate != null) {
+            result.put(INDEX_CREATION_DATE, String.valueOf(lifecycleDate));
+        }
+        if (phaseTime != null) {
+            result.put(PHASE_TIME, String.valueOf(phaseTime));
+        }
+        if (actionTime != null) {
+            result.put(ACTION_TIME, String.valueOf(actionTime));
+        }
+        if (stepTime != null) {
+            result.put(STEP_TIME, String.valueOf(stepTime));
+        }
+        if (phaseDefinition != null) {
+            result.put(PHASE_DEFINITION, String.valueOf(phaseDefinition));
+        }
+        return Collections.unmodifiableMap(result);
+    }
+
+    public String getPhase() {
+        return phase;
+    }
+
+    public String getAction() {
+        return action;
+    }
+
+    public String getStep() {
+        return step;
+    }
+
+    public String getFailedStep() {
+        return failedStep;
+    }
+
+    public String getStepInfo() {
+        return stepInfo;
+    }
+
+    public String getPhaseDefinition() {
+        return phaseDefinition;
+    }
+
+    public Long getLifecycleDate() {
+        return lifecycleDate;
+    }
+
+    public Long getPhaseTime() {
+        return phaseTime;
+    }
+
+    public Long getActionTime() {
+        return actionTime;
+    }
+
+    public Long getStepTime() {
+        return stepTime;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        LifecycleExecutionState that = (LifecycleExecutionState) o;
+        return getLifecycleDate() == that.getLifecycleDate() &&
+            getPhaseTime() == that.getPhaseTime() &&
+            getActionTime() == that.getActionTime() &&
+            getStepTime() == that.getStepTime() &&
+            Objects.equals(getPhase(), that.getPhase()) &&
+            Objects.equals(getAction(), that.getAction()) &&
+            Objects.equals(getStep(), that.getStep()) &&
+            Objects.equals(getFailedStep(), that.getFailedStep()) &&
+            Objects.equals(getStepInfo(), that.getStepInfo()) &&
+            Objects.equals(getPhaseDefinition(), that.getPhaseDefinition());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getPhase(), getAction(), getStep(), getFailedStep(), getStepInfo(), getPhaseDefinition(),
+            getLifecycleDate(), getPhaseTime(), getActionTime(), getStepTime());
+    }
+
+    public static class Builder {
+        private String phase;
+        private String action;
+        private String step;
+        private String failedStep;
+        private String stepInfo;
+        private String phaseDefinition;
+        private Long indexCreationDate;
+        private Long phaseTime;
+        private Long actionTime;
+        private Long stepTime;
+
+        public Builder setPhase(String phase) {
+            this.phase = phase;
+            return this;
+        }
+
+        public Builder setAction(String action) {
+            this.action = action;
+            return this;
+        }
+
+        public Builder setStep(String step) {
+            this.step = step;
+            return this;
+        }
+
+        public Builder setFailedStep(String failedStep) {
+            this.failedStep = failedStep;
+            return this;
+        }
+
+        public Builder setStepInfo(String stepInfo) {
+            this.stepInfo = stepInfo;
+            return this;
+        }
+
+        public Builder setPhaseDefinition(String phaseDefinition) {
+            this.phaseDefinition = phaseDefinition;
+            return this;
+        }
+
+        public Builder setIndexCreationDate(Long indexCreationDate) {
+            this.indexCreationDate = indexCreationDate;
+            return this;
+        }
+
+        public Builder setPhaseTime(Long phaseTime) {
+            this.phaseTime = phaseTime;
+            return this;
+        }
+
+        public Builder setActionTime(Long actionTime) {
+            this.actionTime = actionTime;
+            return this;
+        }
+
+        public Builder setStepTime(Long stepTime) {
+            this.stepTime = stepTime;
+            return this;
+        }
+
+        public LifecycleExecutionState build() {
+            return new LifecycleExecutionState(phase, action, step, failedStep, stepInfo, phaseDefinition, indexCreationDate,
+                phaseTime, actionTime, stepTime);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java
new file mode 100644
index 0000000000000..a56818355c3e5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicy.java
@@ -0,0 +1,266 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Represents the lifecycle of an index from creation to deletion. A
+ * {@link LifecyclePolicy} is made up of a set of {@link Phase}s which it will
+ * move through. Policies are constrained by a {@link LifecycleType} which governs which
+ * {@link Phase}s and {@link LifecycleAction}s are allowed to be defined and in which order
+ * they are executed.
+ */
+public class LifecyclePolicy extends AbstractDiffable<LifecyclePolicy>
+        implements ToXContentObject, Diffable<LifecyclePolicy> {
+    private static final Logger logger = LogManager.getLogger(LifecyclePolicy.class);
+
+    public static final ParseField PHASES_FIELD = new ParseField("phases");
+
+    @SuppressWarnings("unchecked")
+    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
+            (a, name) -> {
+                List<Phase> phases = (List<Phase>) a[0];
+                Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));
+                return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phaseMap);
+            });
+    static {
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> Phase.parse(p, n), v -> {
+            throw new IllegalArgumentException("ordered " + PHASES_FIELD.getPreferredName() + " are not supported");
+        }, PHASES_FIELD);
+    }
+
+    private final String name;
+    private final LifecycleType type;
+    private final Map<String, Phase> phases;
+
+    /**
+     * @param name
+     *            the name of this {@link LifecyclePolicy}
+     * @param phases
+     *            a {@link Map} of {@link Phase}s which make up this
+     *            {@link LifecyclePolicy}.
+     */
+    public LifecyclePolicy(String name, Map<String, Phase> phases) {
+        this(TimeseriesLifecycleType.INSTANCE, name, phases);
+    }
+
+    /**
+     * For Serialization
+     */
+    public LifecyclePolicy(StreamInput in) throws IOException {
+        type = in.readNamedWriteable(LifecycleType.class);
+        name = in.readString();
+        phases = Collections.unmodifiableMap(in.readMap(StreamInput::readString, Phase::new));
+    }
+
+    /**
+     * @param type
+     *            the {@link LifecycleType} of the policy
+     * @param name
+     *            the name of this {@link LifecyclePolicy}
+     * @param phases
+     *            a {@link Map} of {@link Phase}s which make up this
+     *            {@link LifecyclePolicy}.
+     */
+    public LifecyclePolicy(LifecycleType type, String name, Map<String, Phase> phases) {
+        this.name = name;
+        this.phases = phases;
+        this.type = type;
+        this.type.validate(phases.values());
+    }
+
+    public static LifecyclePolicy parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(type);
+        out.writeString(name);
+        out.writeMap(phases, StreamOutput::writeString, (o, val) -> val.writeTo(o));
+    }
+
+    /**
+     * @return the name of this {@link LifecyclePolicy}
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return the type of this {@link LifecyclePolicy}
+     */
+    public LifecycleType getType() {
+        return type;
+    }
+
+    /**
+     * @return the {@link Phase}s for this {@link LifecyclePolicy} in the order
+     *         in which they will be executed.
+     */
+    public Map<String, Phase> getPhases() {
+        return phases;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+            builder.startObject(PHASES_FIELD.getPreferredName());
+                for (Phase phase : phases.values()) {
+                    builder.field(phase.getName(), phase);
+                }
+            builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * This method is used to compile this policy into its execution plan built out
+     * of {@link Step} instances. The order of the {@link Phase}s and {@link LifecycleAction}s is
+     * determined by the {@link LifecycleType} associated with this policy.
+     *
+     * The order of the policy will have this structure:
+     *
+     * - initialize policy context step
+     * - phase-1 phase-after-step
+     * - ... phase-1 action steps
+     * - phase-2 phase-after-step
+     * - ...
+     * - terminal policy step
+     *
+     * We first initialize the policy's context and ensure that the index has proper settings set.
+     * Then we begin each phase's after-step along with all its actions as steps. Finally, we have
+     * a terminal step to inform us that this policy's steps are all complete. Each phase's `after`
+     * step is associated with the previous phase's phase. For example, the warm phase's `after` is
+     * associated with the hot phase so that it is clear that we haven't stepped into the warm phase
+     * just yet (until this step is complete).
+     *
+     * @param client The Elasticsearch Client to use during execution of {@link AsyncActionStep}
+     *               and {@link AsyncWaitStep} steps.
+     * @return The list of {@link Step} objects in order of their execution.
+     */
+    public List<Step> toSteps(Client client) {
+        List<Step> steps = new ArrayList<>();
+        List<Phase> orderedPhases = type.getOrderedPhases(phases);
+        ListIterator<Phase> phaseIterator = orderedPhases.listIterator(orderedPhases.size());
+
+        // final step so that policy can properly update cluster-state with last action completed
+        steps.add(TerminalPolicyStep.INSTANCE);
+        Step.StepKey lastStepKey = TerminalPolicyStep.KEY;
+
+        Phase phase = null;
+        // add steps for each phase, in reverse
+        while (phaseIterator.hasPrevious()) {
+
+            Phase previousPhase = phaseIterator.previous();
+
+            // add `after` step for phase before next
+            if (phase != null) {
+                // after step should have the name of the previous phase since the index is still in the
+                // previous phase until the after condition is reached
+                Step.StepKey afterStepKey = new Step.StepKey(previousPhase.getName(), PhaseCompleteStep.NAME, PhaseCompleteStep.NAME);
+                Step phaseAfterStep = new PhaseCompleteStep(afterStepKey, lastStepKey);
+                steps.add(phaseAfterStep);
+                lastStepKey = phaseAfterStep.getKey();
+            }
+
+            phase = previousPhase;
+            List<LifecycleAction> orderedActions = type.getOrderedActions(phase);
+            ListIterator<LifecycleAction> actionIterator = orderedActions.listIterator(orderedActions.size());
+            // add steps for each action, in reverse
+            while (actionIterator.hasPrevious()) {
+                LifecycleAction action = actionIterator.previous();
+                List<Step> actionSteps = action.toSteps(client, phase.getName(), lastStepKey);
+                ListIterator<Step> actionStepsIterator = actionSteps.listIterator(actionSteps.size());
+                while (actionStepsIterator.hasPrevious()) {
+                    Step step = actionStepsIterator.previous();
+                    steps.add(step);
+                    lastStepKey = step.getKey();
+                }
+            }
+        }
+
+        if (phase != null) {
+            // The very first after step is in a phase before the hot phase so call this "new"
+            Step.StepKey afterStepKey = new Step.StepKey("new", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME);
+            Step phaseAfterStep = new PhaseCompleteStep(afterStepKey, lastStepKey);
+            steps.add(phaseAfterStep);
+            lastStepKey = phaseAfterStep.getKey();
+        }
+
+        // init step so that policy is guaranteed to have
+        steps.add(new InitializePolicyContextStep(InitializePolicyContextStep.KEY, lastStepKey));
+
+        Collections.reverse(steps);
+
+        return steps;
+    }
+
+    public boolean isActionSafe(StepKey stepKey) {
+        if ("new".equals(stepKey.getPhase())) {
+            return true;
+        }
+        Phase phase = phases.get(stepKey.getPhase());
+        if (phase != null) {
+            LifecycleAction action = phase.getActions().get(stepKey.getAction());
+            if (action != null) {
+                return action.isSafeAction();
+            } else {
+                throw new IllegalArgumentException("Action [" + stepKey.getAction() + "] in phase [" + stepKey.getPhase()
+                        + "]  does not exist in policy [" + name + "]");
+            }
+        } else {
+            throw new IllegalArgumentException("Phase [" + stepKey.getPhase() + "]  does not exist in policy [" + name + "]");
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, phases);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        LifecyclePolicy other = (LifecyclePolicy) obj;
+        return Objects.equals(name, other.name) &&
+                Objects.equals(phases, other.phases);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadata.java
new file mode 100644
index 0000000000000..bfb2bee1edaac
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadata.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Map;
+import java.util.Objects;
+
+public class LifecyclePolicyMetadata extends AbstractDiffable<LifecyclePolicyMetadata>
+        implements ToXContentObject, Diffable<LifecyclePolicyMetadata> {
+
+    static final ParseField POLICY = new ParseField("policy");
+    static final ParseField HEADERS = new ParseField("headers");
+    static final ParseField VERSION = new ParseField("version");
+    static final ParseField MODIFIED_DATE = new ParseField("modified_date");
+    static final ParseField MODIFIED_DATE_STRING = new ParseField("modified_date_string");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
+            a -> {
+                LifecyclePolicy policy = (LifecyclePolicy) a[0];
+                return new LifecyclePolicyMetadata(policy, (Map<String, String>) a[1], (long) a[2], (long) a[3]);
+            });
+    static {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), LifecyclePolicy::parse, POLICY);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), XContentParser::mapStrings, HEADERS, ValueType.OBJECT);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_DATE);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_STRING);
+    }
+
+    public static LifecyclePolicyMetadata parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final LifecyclePolicy policy;
+    private final Map<String, String> headers;
+    private final long version;
+    private final long modifiedDate;
+
+    public LifecyclePolicyMetadata(LifecyclePolicy policy, Map<String, String> headers, long version, long modifiedDate) {
+        this.policy = policy;
+        this.headers = headers;
+        this.version = version;
+        this.modifiedDate = modifiedDate;
+    }
+
+    @SuppressWarnings("unchecked")
+    public LifecyclePolicyMetadata(StreamInput in) throws IOException {
+        this.policy = new LifecyclePolicy(in);
+        this.headers = (Map<String, String>) in.readGenericValue();
+        this.version = in.readVLong();
+        this.modifiedDate = in.readVLong();
+    }
+
+    public Map<String, String> getHeaders() {
+        return headers;
+    }
+
+    public LifecyclePolicy getPolicy() {
+        return policy;
+    }
+
+    public String getName() {
+        return policy.getName();
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public long getModifiedDate() {
+        return modifiedDate;
+    }
+
+    public String getModifiedDateString() {
+        ZonedDateTime modifiedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(modifiedDate), ZoneOffset.UTC);
+        return modifiedDateTime.toString();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(POLICY.getPreferredName(), policy);
+        builder.field(HEADERS.getPreferredName(), headers);
+        builder.field(VERSION.getPreferredName(), version);
+        builder.field(MODIFIED_DATE.getPreferredName(), modifiedDate);
+        builder.field(MODIFIED_DATE_STRING.getPreferredName(), getModifiedDateString());
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        policy.writeTo(out);
+        out.writeGenericValue(headers);
+        out.writeVLong(version);
+        out.writeVLong(modifiedDate);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(policy, headers, version, modifiedDate);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        LifecyclePolicyMetadata other = (LifecyclePolicyMetadata) obj;
+        return Objects.equals(policy, other.policy) &&
+            Objects.equals(headers, other.headers) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(modifiedDate, other.modifiedDate);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java
new file mode 100644
index 0000000000000..4f8eb339db7e8
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleSettings.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.TimeValue;
+
+/**
+ * Class encapsulating settings related to Index Lifecycle Management X-Pack Plugin
+ */
+public class LifecycleSettings {
+    public static final String LIFECYCLE_POLL_INTERVAL = "indices.lifecycle.poll_interval";
+    public static final String LIFECYCLE_NAME = "index.lifecycle.name";
+
+    public static final Setting<TimeValue> LIFECYCLE_POLL_INTERVAL_SETTING = Setting.positiveTimeSetting(LIFECYCLE_POLL_INTERVAL,
+        TimeValue.timeValueMinutes(10), Setting.Property.Dynamic, Setting.Property.NodeScope);
+    public static final Setting<String> LIFECYCLE_NAME_SETTING = Setting.simpleString(LIFECYCLE_NAME,
+        Setting.Property.Dynamic, Setting.Property.IndexScope);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleType.java
new file mode 100644
index 0000000000000..69be30fdfbd0e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleType.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+
+import java.security.Policy;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface LifecycleType extends NamedWriteable {
+
+    /**
+     * @return the first phase of this policy to execute
+     */
+    List<Phase> getOrderedPhases(Map<String, Phase> phases);
+
+    /**
+     * Returns the next phase thats available after
+     * <code>currentPhaseName</code>. Note that <code>currentPhaseName</code>
+     * does not need to exist in <code>phases</code>.
+     * 
+     * If the current {@link Phase} is the last phase in the {@link Policy} this
+     * method will return <code>null</code>.
+     * 
+     * If the phase is not valid for the lifecycle type an
+     * {@link IllegalArgumentException} will be thrown.
+     */
+    String getNextPhaseName(String currentPhaseName, Map<String, Phase> phases);
+
+    /**
+     * Returns the previous phase thats available before
+     * <code>currentPhaseName</code>. Note that <code>currentPhaseName</code>
+     * does not need to exist in <code>phases</code>.
+     * 
+     * If the current {@link Phase} is the first phase in the {@link Policy}
+     * this method will return <code>null</code>.
+     * 
+     * If the phase is not valid for the lifecycle type an
+     * {@link IllegalArgumentException} will be thrown.
+     */
+    String getPreviousPhaseName(String currentPhaseName, Map<String, Phase> phases);
+
+    List<LifecycleAction> getOrderedActions(Phase phase);
+
+    /**
+     * Returns the name of the next phase that is available in the phases after
+     * <code>currentActionName</code>. Note that <code>currentActionName</code>
+     * does not need to exist in the {@link Phase}.
+     * 
+     * If the current action is the last action in the phase this method will
+     * return <code>null</code>.
+     * 
+     * If the action is not valid for the phase an
+     * {@link IllegalArgumentException} will be thrown.
+     */
+    String getNextActionName(String currentActionName, Phase phase);
+
+
+    /**
+     * validates whether the specified <code>phases</code> are valid for this
+     * policy instance.
+     * 
+     * @param phases
+     *            the phases to verify validity against
+     * @throws IllegalArgumentException
+     *             if a specific phase or lack of a specific phase is invalid.
+     */
+    void validate(Collection<Phase> phases);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OperationMode.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OperationMode.java
new file mode 100644
index 0000000000000..defc2e46818bc
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/OperationMode.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.admin.indices.shrink.ShrinkAction;
+
+/**
+ * Enum representing the different modes that Index Lifecycle Service can operate in.
+ */
+public enum OperationMode {
+    /**
+     * This represents a state where no policies are executed
+     */
+    STOPPED {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == RUNNING;
+        }
+    },
+
+    /**
+     * this represents a state where only sensitive actions (like {@link ShrinkAction}) will be executed
+     * until they finish, at which point the operation mode will move to <code>STOPPED</code>.
+     */
+    STOPPING {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == RUNNING || nextMode == STOPPED;
+        }
+    },
+
+    /**
+     * Normal operation where all policies are executed as normal.
+     */
+    RUNNING {
+        @Override
+        public boolean isValidChange(OperationMode nextMode) {
+            return nextMode == STOPPING;
+        }
+    };
+
+    public abstract boolean isValidChange(OperationMode nextMode);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Phase.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Phase.java
new file mode 100644
index 0000000000000..08b995ade1460
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Phase.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Represents set of {@link LifecycleAction}s which should be executed at a
+ * particular point in the lifecycle of an index.
+ */
+public class Phase implements ToXContentObject, Writeable {
+
+    public static final ParseField MIN_AGE = new ParseField("min_age");
+    public static final ParseField ACTIONS_FIELD = new ParseField("actions");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", false,
+            (a, name) -> new Phase(name, (TimeValue) a[0], ((List<LifecycleAction>) a[1]).stream()
+                    .collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()))));
+    static {
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+                (p, c) -> TimeValue.parseTimeValue(p.text(), MIN_AGE.getPreferredName()), MIN_AGE, ValueType.VALUE);
+        PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(),
+                (p, c, n) -> p.namedObject(LifecycleAction.class, n, null), v -> {
+                    throw new IllegalArgumentException("ordered " + ACTIONS_FIELD.getPreferredName() + " are not supported");
+                }, ACTIONS_FIELD);
+    }
+
+    public static Phase parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final String name;
+    private final Map<String, LifecycleAction> actions;
+    private final TimeValue minimumAge;
+
+    /**
+     * @param name
+     *            the name of this {@link Phase}.
+     * @param minimumAge
+     *            the age of the index when the index should move to this
+     *            {@link Phase}.
+     * @param actions
+     *            a {@link Map} of the {@link LifecycleAction}s to run when
+     *            during his {@link Phase}. The keys in this map are the associated
+     *            action names. The order of these actions is defined
+     *            by the {@link LifecycleType}
+     */
+    public Phase(String name, TimeValue minimumAge, Map<String, LifecycleAction> actions) {
+        this.name = name;
+        if (minimumAge == null) {
+            this.minimumAge = TimeValue.ZERO;
+        } else {
+            this.minimumAge = minimumAge;
+        }
+        this.actions = actions;
+    }
+
+    /**
+     * For Serialization
+     */
+    public Phase(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.minimumAge = in.readTimeValue();
+        int size = in.readVInt();
+        TreeMap<String, LifecycleAction> actions = new TreeMap<>();
+        for (int i = 0; i < size; i++) {
+            actions.put(in.readString(), in.readNamedWriteable(LifecycleAction.class));
+        }
+        this.actions = actions;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeTimeValue(minimumAge);
+        out.writeVInt(actions.size());
+        for (Map.Entry<String, LifecycleAction> entry : actions.entrySet()) {
+            out.writeString(entry.getKey());
+            out.writeNamedWriteable(entry.getValue());
+        }
+    }
+
+    /**
+     * @return the age of the index when the index should move to this
+     *         {@link Phase}.
+     */
+    public TimeValue getMinimumAge() {
+        return minimumAge;
+    }
+
+    /**
+     * @return the name of this {@link Phase}
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return a {@link Map} of the {@link LifecycleAction}s to run when during
+     *         his {@link Phase}.
+     */
+    public Map<String, LifecycleAction> getActions() {
+        return actions;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(MIN_AGE.getPreferredName(), minimumAge.getStringRep());
+        builder.field(ACTIONS_FIELD.getPreferredName(), actions);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, minimumAge, actions);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        Phase other = (Phase) obj;
+        return Objects.equals(name, other.name) &&
+                Objects.equals(minimumAge, other.minimumAge) &&
+                Objects.equals(actions, other.actions);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStep.java
new file mode 100644
index 0000000000000..fa960bb9994b4
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStep.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+/**
+ * This is essentially a marker that a phase has ended, and we need to check
+ * the age of an index before proceeding to the next phase.
+ */
+public class PhaseCompleteStep extends Step {
+    public static final String NAME = "complete";
+
+    public PhaseCompleteStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfo.java
new file mode 100644
index 0000000000000..1ba7390ed2202
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfo.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * This class contains information about the current phase being executed by Index
+ * Lifecycle Management on the specific index.
+ */
+public class PhaseExecutionInfo implements ToXContentObject, Writeable {
+    private static final ParseField POLICY_NAME_FIELD = new ParseField("policy");
+    private static final ParseField PHASE_DEFINITION_FIELD = new ParseField("phase_definition");
+    private static final ParseField VERSION_FIELD = new ParseField("version");
+    private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis");
+
+    private static final ConstructingObjectParser<PhaseExecutionInfo, String> PARSER = new ConstructingObjectParser<>(
+        "phase_execution_info", false,
+        (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3]));
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Phase::parse, PHASE_DEFINITION_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), VERSION_FIELD);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_DATE_IN_MILLIS_FIELD);
+    }
+
+    public static PhaseExecutionInfo parse(XContentParser parser, String name) {
+        return PARSER.apply(parser, name);
+    }
+
+    private final String policyName;
+    private final Phase phase;
+    private final long version;
+    private final long modifiedDate;
+
+    /**
+     * This class holds information about the current phase that is being executed
+     *
+     * @param policyName the name of the policy being executed, this may not be the current policy assigned to an index
+     * @param phase the current phase definition executed
+     * @param version the version of the <code>policyName</code> being executed
+     * @param modifiedDate the time the executing version of the phase was modified
+     */
+    public PhaseExecutionInfo(String policyName, @Nullable Phase phase, long version, long modifiedDate) {
+        this.policyName = policyName;
+        this.phase = phase;
+        this.version = version;
+        this.modifiedDate = modifiedDate;
+    }
+
+    PhaseExecutionInfo(StreamInput in) throws IOException {
+        this.policyName = in.readString();
+        this.phase = in.readOptionalWriteable(Phase::new);
+        this.version = in.readVLong();
+        this.modifiedDate = in.readVLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(policyName);
+        out.writeOptionalWriteable(phase);
+        out.writeVLong(version);
+        out.writeVLong(modifiedDate);
+    }
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    public Phase getPhase() {
+        return phase;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public long getModifiedDate() {
+        return modifiedDate;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(policyName, phase, version, modifiedDate);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        PhaseExecutionInfo other = (PhaseExecutionInfo) obj;
+        return Objects.equals(policyName, other.policyName) &&
+            Objects.equals(phase, other.phase) &&
+            Objects.equals(version, other.version) &&
+            Objects.equals(modifiedDate, other.modifiedDate);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, false, true);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(POLICY_NAME_FIELD.getPreferredName(), policyName);
+        if (phase != null) {
+            builder.field(PHASE_DEFINITION_FIELD.getPreferredName(), phase);
+        }
+        builder.field(VERSION_FIELD.getPreferredName(), version);
+        builder.timeField(MODIFIED_DATE_IN_MILLIS_FIELD.getPreferredName(), "modified_date", modifiedDate);
+        builder.endObject();
+        return builder;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyAction.java
new file mode 100644
index 0000000000000..15edd51908bfe
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyAction.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A {@link LifecycleAction} which force-merges the index.
+ */
+public class ReadOnlyAction implements LifecycleAction {
+    public static final String NAME = "readonly";
+    public static final ReadOnlyAction INSTANCE = new ReadOnlyAction();
+
+    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
+
+    public static ReadOnlyAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ReadOnlyAction() {
+    }
+
+    public ReadOnlyAction(StreamInput in) {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return true;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        Step.StepKey key = new Step.StepKey(phase, NAME, NAME);
+        Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
+        return Collections.singletonList(new UpdateSettingsStep(key, nextStepKey, client, readOnlySettings));
+    }
+    
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        return Collections.singletonList(new Step.StepKey(phase, NAME, NAME));
+    }
+
+    @Override
+    public int hashCode() {
+        return ReadOnlyAction.class.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java
new file mode 100644
index 0000000000000..78dce2db1b8c2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverAction.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A {@link LifecycleAction} which deletes the index.
+ */
+public class RolloverAction implements LifecycleAction {
+    public static final String NAME = "rollover";
+    public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
+    public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
+    public static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
+    public static final String LIFECYCLE_ROLLOVER_ALIAS = "index.lifecycle.rollover_alias";
+    public static final Setting<String> LIFECYCLE_ROLLOVER_ALIAS_SETTING = Setting.simpleString(LIFECYCLE_ROLLOVER_ALIAS,
+        Setting.Property.Dynamic, Setting.Property.IndexScope);
+
+    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+            a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
+    static {
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+                (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
+                (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
+    }
+
+    private final ByteSizeValue maxSize;
+    private final Long maxDocs;
+    private final TimeValue maxAge;
+
+    public static RolloverAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public RolloverAction(ByteSizeValue maxSize, TimeValue maxAge, Long maxDocs) {
+        if (maxSize == null && maxAge == null && maxDocs == null) {
+            throw new IllegalArgumentException("At least one rollover condition must be set.");
+        }
+        this.maxSize = maxSize;
+        this.maxAge = maxAge;
+        this.maxDocs = maxDocs;
+    }
+
+    public RolloverAction(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            maxSize = new ByteSizeValue(in);
+        } else {
+            maxSize = null;
+        }
+        maxAge = in.readOptionalTimeValue();
+        if (in.readBoolean()) {
+            maxDocs = in.readVLong();
+        } else {
+            maxDocs = null;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        boolean hasMaxSize = maxSize != null;
+        out.writeBoolean(hasMaxSize);
+        if (hasMaxSize) {
+            maxSize.writeTo(out);
+        }
+        out.writeOptionalTimeValue(maxAge);
+        boolean hasMaxDocs = maxDocs != null;
+        out.writeBoolean(hasMaxDocs);
+        if (hasMaxDocs) {
+            out.writeVLong(maxDocs);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public ByteSizeValue getMaxSize() {
+        return maxSize;
+    }
+
+    public TimeValue getMaxAge() {
+        return maxAge;
+    }
+
+    public Long getMaxDocs() {
+        return maxDocs;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (maxSize != null) {
+            builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
+        }
+        if (maxAge != null) {
+            builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
+        }
+        if (maxDocs != null) {
+            builder.field(MAX_DOCS_FIELD.getPreferredName(), maxDocs);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return true;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME);
+        RolloverStep rolloverStep = new RolloverStep(new StepKey(phase, NAME, RolloverStep.NAME), updateDateStepKey, client,
+            maxSize, maxAge, maxDocs);
+        UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, nextStepKey);
+        return Arrays.asList(rolloverStep, updateDateStep);
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        StepKey rolloverStepKey = new StepKey(phase, NAME, RolloverStep.NAME);
+        StepKey updateDateStepKey = new StepKey(phase, NAME, UpdateRolloverLifecycleDateStep.NAME);
+        return Arrays.asList(rolloverStepKey, updateDateStepKey);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        RolloverAction other = (RolloverAction) obj;
+        return Objects.equals(maxSize, other.maxSize) &&
+                Objects.equals(maxAge, other.maxAge) &&
+                Objects.equals(maxDocs, other.maxDocs);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java
new file mode 100644
index 0000000000000..399f90df31dae
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStep.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class RolloverStep extends AsyncWaitStep {
+    public static final String NAME = "attempt_rollover";
+
+    private static final Logger logger = LogManager.getLogger(RolloverStep.class);
+
+    private ByteSizeValue maxSize;
+    private TimeValue maxAge;
+    private Long maxDocs;
+
+    public RolloverStep(StepKey key, StepKey nextStepKey, Client client, ByteSizeValue maxSize, TimeValue maxAge,
+            Long maxDocs) {
+        super(key, nextStepKey, client);
+        this.maxSize = maxSize;
+        this.maxAge = maxAge;
+        this.maxDocs = maxDocs;
+    }
+
+    @Override
+    public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
+        String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings());
+
+        if (Strings.isNullOrEmpty(rolloverAlias)) {
+            listener.onFailure(new IllegalArgumentException(String.format(Locale.ROOT,
+                "setting [%s] for index [%s] is empty or not defined", RolloverAction.LIFECYCLE_ROLLOVER_ALIAS,
+                indexMetaData.getIndex().getName())));
+            return;
+        }
+
+        if (indexMetaData.getAliases().containsKey(rolloverAlias) == false) {
+            listener.onFailure(new IllegalArgumentException(String.format(Locale.ROOT,
+                "%s [%s] does not point to index [%s]", RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, rolloverAlias,
+                indexMetaData.getIndex().getName())));
+            return;
+        }
+
+        RolloverRequest rolloverRequest = new RolloverRequest(rolloverAlias, null);
+        if (maxAge != null) {
+            rolloverRequest.addMaxIndexAgeCondition(maxAge);
+        }
+        if (maxSize != null) {
+            rolloverRequest.addMaxIndexSizeCondition(maxSize);
+        }
+        if (maxDocs != null) {
+            rolloverRequest.addMaxIndexDocsCondition(maxDocs);
+        }
+        getClient().admin().indices().rolloverIndex(rolloverRequest,
+            ActionListener.wrap(response -> listener.onResponse(response.isRolledOver(), new EmptyInfo()), exception -> {
+                if (exception instanceof ResourceAlreadyExistsException) {
+                    // This can happen sometimes when this step is executed multiple times
+                    if (logger.isTraceEnabled()) {
+                        logger.debug(() -> new ParameterizedMessage("{} index cannot roll over because the next index already exists, " +
+                            "skipping to next step", indexMetaData.getIndex()), exception);
+                    } else {
+                        logger.debug("{} index cannot roll over because the next index already exists, skipping to next step",
+                            indexMetaData.getIndex());
+                    }
+                    listener.onResponse(true, new EmptyInfo());
+                } else {
+                    listener.onFailure(exception);
+                }
+            }));
+    }
+
+    ByteSizeValue getMaxSize() {
+        return maxSize;
+    }
+
+    TimeValue getMaxAge() {
+        return maxAge;
+    }
+
+    Long getMaxDocs() {
+        return maxDocs;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        RolloverStep other = (RolloverStep) obj;
+        return super.equals(obj) &&
+                Objects.equals(maxSize, other.maxSize) &&
+                Objects.equals(maxAge, other.maxAge) &&
+                Objects.equals(maxDocs, other.maxDocs);
+    }
+
+    // We currently have no information to provide for this AsyncWaitStep, so this is an empty object
+    private class EmptyInfo implements ToXContentObject {
+        private EmptyInfo() {}
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java
new file mode 100644
index 0000000000000..0d706dca10445
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.StreamSupport;
+
+/**
+ * This {@link Step} evaluates whether force_merge was successful
+ */
+public class SegmentCountStep extends AsyncWaitStep {
+    public static final String NAME = "segment-count";
+
+    private final int maxNumSegments;
+
+    public SegmentCountStep(StepKey key, StepKey nextStepKey, Client client, int maxNumSegments) {
+        super(key, nextStepKey, client);
+        this.maxNumSegments = maxNumSegments;
+    }
+
+    public int getMaxNumSegments() {
+        return maxNumSegments;
+    }
+
+    @Override
+    public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
+        getClient().admin().indices().segments(new IndicesSegmentsRequest(indexMetaData.getIndex().getName()),
+            ActionListener.wrap(response -> {
+                long numberShardsLeftToMerge =
+                    StreamSupport.stream(response.getIndices().get(indexMetaData.getIndex().getName()).spliterator(), false)
+                        .filter(iss -> Arrays.stream(iss.getShards()).anyMatch(p -> p.getSegments().size() > maxNumSegments)).count();
+                listener.onResponse(numberShardsLeftToMerge == 0, new Info(numberShardsLeftToMerge));
+            }, listener::onFailure));
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), maxNumSegments);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        SegmentCountStep other = (SegmentCountStep) obj;
+        return super.equals(obj)
+            && Objects.equals(maxNumSegments, other.maxNumSegments);
+    }
+
+    public static class Info implements ToXContentObject {
+
+        private final long numberShardsLeftToMerge;
+
+        static final ParseField SHARDS_TO_MERGE = new ParseField("shards_left_to_merge");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<Info, Void> PARSER = new ConstructingObjectParser<>("segment_count_step_info",
+                a -> new Info((long) a[0]));
+        static {
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), SHARDS_TO_MERGE);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(long numberShardsLeftToMerge) {
+            this.numberShardsLeftToMerge = numberShardsLeftToMerge;
+        }
+
+        public long getNumberShardsLeftToMerge() {
+            return numberShardsLeftToMerge;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(),
+                    "Waiting for [" + numberShardsLeftToMerge + "] shards " + "to forcemerge");
+            builder.field(SHARDS_TO_MERGE.getPreferredName(), numberShardsLeftToMerge);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(numberShardsLeftToMerge);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Info other = (Info) obj;
+            return Objects.equals(numberShardsLeftToMerge, other.numberShardsLeftToMerge);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java
new file mode 100644
index 0000000000000..0caa217841769
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+public class SetSingleNodeAllocateStep extends AsyncActionStep {
+    public static final String NAME = "set-single-node-allocation";
+
+    private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(Collections.singletonList(
+            new FilterAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
+
+    public SetSingleNodeAllocateStep(StepKey key, StepKey nextStepKey, Client client) {
+        super(key, nextStepKey, client);
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState clusterState, Listener listener) {
+        RoutingAllocation allocation = new RoutingAllocation(ALLOCATION_DECIDERS, clusterState.getRoutingNodes(), clusterState, null,
+                System.nanoTime());
+        List<String> validNodeIds = new ArrayList<>();
+        Optional<ShardRouting> anyShard = clusterState.getRoutingTable().allShards(indexMetaData.getIndex().getName()).stream().findAny();
+        if (anyShard.isPresent()) {
+            // Iterate through the nodes finding ones that are acceptable for the current allocation rules of the shard
+            for (RoutingNode node : clusterState.getRoutingNodes()) {
+                boolean canRemainOnCurrentNode = ALLOCATION_DECIDERS.canRemain(anyShard.get(), node, allocation)
+                        .type() == Decision.Type.YES;
+                if (canRemainOnCurrentNode) {
+                    DiscoveryNode discoveryNode = node.node();
+                    validNodeIds.add(discoveryNode.getId());
+                }
+            }
+            // Shuffle the list of nodes so the one we pick is random
+            Randomness.shuffle(validNodeIds);
+            Optional<String> nodeId = validNodeIds.stream().findAny();
+            if (nodeId.isPresent()) {
+                Settings settings = Settings.builder()
+                        .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", nodeId.get()).build();
+                UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexMetaData.getIndex().getName())
+                        .settings(settings);
+                getClient().admin().indices().updateSettings(updateSettingsRequest,
+                        ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
+            } else {
+                // No nodes currently match the allocation rules so just wait until there is one that does
+                listener.onResponse(false);
+            }
+        } else {
+            // There are no shards for the index, the index might be gone
+            listener.onFailure(new IndexNotFoundException(indexMetaData.getIndex()));
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java
new file mode 100644
index 0000000000000..a79383c24de8b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A {@link LifecycleAction} which shrinks the index.
+ */
+public class ShrinkAction implements LifecycleAction {
+    public static final String NAME = "shrink";
+    public static final String SHRUNKEN_INDEX_PREFIX = "shrink-";
+    public static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards");
+
+    private static final ConstructingObjectParser<ShrinkAction, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD);
+    }
+
+    private int numberOfShards;
+
+    public static ShrinkAction parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public ShrinkAction(int numberOfShards) {
+        if (numberOfShards <= 0) {
+            throw new IllegalArgumentException("[" + NUMBER_OF_SHARDS_FIELD.getPreferredName() + "] must be greater than 0");
+        }
+        this.numberOfShards = numberOfShards;
+    }
+
+    public ShrinkAction(StreamInput in) throws IOException {
+        this.numberOfShards = in.readVInt();
+    }
+
+    int getNumberOfShards() {
+        return numberOfShards;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(numberOfShards);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(NUMBER_OF_SHARDS_FIELD.getPreferredName(), numberOfShards);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return false;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
+
+        StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
+        StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
+        StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
+        StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
+        StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
+        StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
+        StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
+        StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
+
+        UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
+        SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
+        CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey);
+        ShrinkStep shrink = new ShrinkStep(shrinkKey, enoughShardsKey, client, numberOfShards, SHRUNKEN_INDEX_PREFIX);
+        ShrunkShardsAllocatedStep allocated = new ShrunkShardsAllocatedStep(enoughShardsKey, copyMetadataKey, SHRUNKEN_INDEX_PREFIX);
+        CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
+        ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
+        ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
+        return Arrays.asList(readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated, copyMetadata,
+            aliasSwapAndDelete, waitOnShrinkTakeover);
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
+        StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
+        StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
+        StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
+        StepKey enoughShardsKey = new StepKey(phase, NAME, ShrunkShardsAllocatedStep.NAME);
+        StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
+        StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
+        StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
+        return Arrays.asList(readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
+            copyMetadataKey, aliasKey, isShrunkIndexKey);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ShrinkAction that = (ShrinkAction) o;
+        return Objects.equals(numberOfShards, that.numberOfShards);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(numberOfShards);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStep.java
new file mode 100644
index 0000000000000..b9e0e00eeb600
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStep.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+
+import java.util.Objects;
+
+public class ShrinkSetAliasStep extends AsyncActionStep {
+    public static final String NAME = "aliases";
+    private String shrunkIndexPrefix;
+
+    public ShrinkSetAliasStep(StepKey key, StepKey nextStepKey, Client client, String shrunkIndexPrefix) {
+        super(key, nextStepKey, client);
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+        // get source index
+        String index = indexMetaData.getIndex().getName();
+        // get target shrink index
+        String targetIndexName = shrunkIndexPrefix + index;
+
+        IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest()
+            .addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index(index))
+            .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(targetIndexName).alias(index));
+
+        getClient().admin().indices().aliases(aliasesRequest, ActionListener.wrap(response ->
+            listener.onResponse(true), listener::onFailure));
+    }
+
+    @Override
+    public boolean indexSurvives() {
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ShrinkSetAliasStep other = (ShrinkSetAliasStep) obj;
+        return super.equals(obj) && 
+                Objects.equals(shrunkIndexPrefix, other.shrunkIndexPrefix);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStep.java
new file mode 100644
index 0000000000000..8eeaed82e4c12
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStep.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.Objects;
+
+public class ShrinkStep extends AsyncActionStep {
+    public static final String NAME = "shrink";
+
+    private int numberOfShards;
+    private String shrunkIndexPrefix;
+
+    public ShrinkStep(StepKey key, StepKey nextStepKey, Client client, int numberOfShards, String shrunkIndexPrefix) {
+        super(key, nextStepKey, client);
+        this.numberOfShards = numberOfShards;
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    public int getNumberOfShards() {
+        return numberOfShards;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        if (lifecycleState.getLifecycleDate() == null) {
+            throw new IllegalStateException("source index [" + indexMetaData.getIndex().getName() +
+                "] is missing lifecycle date");
+        }
+
+        String lifecycle = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetaData.getSettings());
+
+        Settings relevantTargetSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, indexMetaData.getNumberOfReplicas())
+            .put(LifecycleSettings.LIFECYCLE_NAME, lifecycle)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", (String) null) // need to remove the single shard
+                                                                                             // allocation so replicas can be allocated
+            .build();
+
+        String shrunkenIndexName = shrunkIndexPrefix + indexMetaData.getIndex().getName();
+        ResizeRequest resizeRequest = new ResizeRequest(shrunkenIndexName, indexMetaData.getIndex().getName());
+        resizeRequest.setCopySettings(true);
+        indexMetaData.getAliases().values().spliterator().forEachRemaining(aliasMetaDataObjectCursor -> {
+            resizeRequest.getTargetIndexRequest().alias(new Alias(aliasMetaDataObjectCursor.value.alias()));
+        });
+        resizeRequest.getTargetIndexRequest().settings(relevantTargetSettings);
+
+        getClient().admin().indices().resizeIndex(resizeRequest, ActionListener.wrap(response -> {
+            // TODO(talevy): when is this not acknowledged?
+            listener.onResponse(response.isAcknowledged());
+        }, listener::onFailure));
+
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), numberOfShards, shrunkIndexPrefix);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ShrinkStep other = (ShrinkStep) obj;
+        return super.equals(obj) &&
+                Objects.equals(numberOfShards, other.numberOfShards) &&
+                Objects.equals(shrunkIndexPrefix, other.shrunkIndexPrefix);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStep.java
new file mode 100644
index 0000000000000..b64ebf5e46b5a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStep.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ShrunkShardsAllocatedStep extends ClusterStateWaitStep {
+    public static final String NAME = "shrunk-shards-allocated";
+    private String shrunkIndexPrefix;
+
+    public ShrunkShardsAllocatedStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
+        super(key, nextStepKey);
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public Result isConditionMet(Index index, ClusterState clusterState) {
+        // We only want to make progress if all shards of the shrunk index are
+        // active
+        boolean indexExists = clusterState.metaData().index(shrunkIndexPrefix + index.getName()) != null;
+        if (indexExists == false) {
+            return new Result(false, new Info(false, -1, false));
+        }
+        boolean allShardsActive = ActiveShardCount.ALL.enoughShardsActive(clusterState, shrunkIndexPrefix + index.getName());
+        int numShrunkIndexShards = clusterState.metaData().index(shrunkIndexPrefix + index.getName()).getNumberOfShards();
+        if (allShardsActive) {
+            return new Result(true, null);
+        } else {
+            return new Result(false, new Info(true, numShrunkIndexShards, allShardsActive));
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ShrunkShardsAllocatedStep other = (ShrunkShardsAllocatedStep) obj;
+        return super.equals(obj) && Objects.equals(shrunkIndexPrefix, other.shrunkIndexPrefix);
+    }
+
+    public static final class Info implements ToXContentObject {
+
+        private final int actualShards;
+        private final boolean shrunkIndexExists;
+        private final boolean allShardsActive;
+        private final String message;
+
+        static final ParseField ACTUAL_SHARDS = new ParseField("actual_shards");
+        static final ParseField SHRUNK_INDEX_EXISTS = new ParseField("shrunk_index_exists");
+        static final ParseField ALL_SHARDS_ACTIVE = new ParseField("all_shards_active");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<Info, Void> PARSER = new ConstructingObjectParser<>("shrunk_shards_allocated_step_info",
+                a -> new Info((boolean) a[0], (int) a[1], (boolean) a[2]));
+        static {
+            PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SHRUNK_INDEX_EXISTS);
+            PARSER.declareInt(ConstructingObjectParser.constructorArg(), ACTUAL_SHARDS);
+            PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ALL_SHARDS_ACTIVE);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(boolean shrunkIndexExists, int actualShards, boolean allShardsActive) {
+            this.actualShards = actualShards;
+            this.shrunkIndexExists = shrunkIndexExists;
+            this.allShardsActive = allShardsActive;
+            if (shrunkIndexExists == false) {
+                message = "Waiting for shrunk index to be created";
+            } else if (allShardsActive == false) {
+                message = "Waiting for all shard copies to be active";
+            } else {
+                message = "";
+            }
+        }
+
+        public int getActualShards() {
+            return actualShards;
+        }
+
+        public boolean shrunkIndexExists() {
+            return shrunkIndexExists;
+        }
+
+        public boolean allShardsActive() {
+            return allShardsActive;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(), message);
+            builder.field(SHRUNK_INDEX_EXISTS.getPreferredName(), shrunkIndexExists);
+            builder.field(ACTUAL_SHARDS.getPreferredName(), actualShards);
+            builder.field(ALL_SHARDS_ACTIVE.getPreferredName(), allShardsActive);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(shrunkIndexExists, actualShards, allShardsActive);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Info other = (Info) obj;
+            return Objects.equals(shrunkIndexExists, other.shrunkIndexExists) &&
+                    Objects.equals(actualShards, other.actualShards) &&
+                    Objects.equals(allShardsActive, other.allShardsActive);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStep.java
new file mode 100644
index 0000000000000..201e27b561547
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStep.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ShrunkenIndexCheckStep extends ClusterStateWaitStep {
+    public static final String NAME = "is-shrunken-index";
+    private static final Logger logger = LogManager.getLogger(InitializePolicyContextStep.class);
+    private String shrunkIndexPrefix;
+
+    public ShrunkenIndexCheckStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
+        super(key, nextStepKey);
+        this.shrunkIndexPrefix = shrunkIndexPrefix;
+    }
+
+    String getShrunkIndexPrefix() {
+        return shrunkIndexPrefix;
+    }
+
+    @Override
+    public Result isConditionMet(Index index, ClusterState clusterState) {
+        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        if (idxMeta == null) {
+            logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
+            // Index must have been since deleted, ignore it
+            return new Result(false, null);
+        }
+        String shrunkenIndexSource = IndexMetaData.INDEX_SHRINK_SOURCE_NAME.get(
+            clusterState.metaData().index(index).getSettings());
+        if (Strings.isNullOrEmpty(shrunkenIndexSource)) {
+            throw new IllegalStateException("step[" + NAME + "] is checking an un-shrunken index[" + index.getName() + "]");
+        }
+        boolean isConditionMet = index.getName().equals(shrunkIndexPrefix + shrunkenIndexSource) &&
+                clusterState.metaData().index(shrunkenIndexSource) == null;
+        if (isConditionMet) {
+            return new Result(true, null);
+        } else {
+            return new Result(false, new Info(shrunkenIndexSource));
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shrunkIndexPrefix);
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ShrunkenIndexCheckStep other = (ShrunkenIndexCheckStep) obj;
+        return super.equals(obj) && 
+                Objects.equals(shrunkIndexPrefix, other.shrunkIndexPrefix);
+    }
+
+    public static final class Info implements ToXContentObject {
+
+        private final String originalIndexName;
+        private final String message;
+
+        static final ParseField ORIGINAL_INDEX_NAME = new ParseField("original_index_name");
+        static final ParseField MESSAGE = new ParseField("message");
+        static final ConstructingObjectParser<Info, Void> PARSER = new ConstructingObjectParser<>("shrunken_index_check_step_info",
+                a -> new Info((String) a[0]));
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), ORIGINAL_INDEX_NAME);
+            PARSER.declareString((i, s) -> {}, MESSAGE);
+        }
+
+        public Info(String originalIndexName) {
+            this.originalIndexName = originalIndexName;
+            this.message = "Waiting for original index [" + originalIndexName + "] to be deleted";
+        }
+
+        public String getOriginalIndexName() {
+            return originalIndexName;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(MESSAGE.getPreferredName(), message);
+            builder.field(ORIGINAL_INDEX_NAME.getPreferredName(), originalIndexName);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(originalIndexName);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Info other = (Info) obj;
+            return Objects.equals(originalIndexName, other.originalIndexName);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequest.java
new file mode 100644
index 0000000000000..de38a5e092ae2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+
+public class StartILMRequest extends AcknowledgedRequest<StartILMRequest> {
+
+    public StartILMRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return 64;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java
new file mode 100644
index 0000000000000..7152b093e62ef
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/Step.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A {@link LifecycleAction} which deletes the index.
+ */
+public abstract class Step {
+    private final StepKey key;
+    private final StepKey nextStepKey;
+
+    public Step(StepKey key, StepKey nextStepKey) {
+        this.key = key;
+        this.nextStepKey = nextStepKey;
+    }
+
+    public final StepKey getKey() {
+        return key;
+    }
+
+    public final StepKey getNextStepKey() {
+        return nextStepKey;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, nextStepKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        Step other = (Step) obj;
+        return Objects.equals(key, other.key) &&
+                Objects.equals(nextStepKey, other.nextStepKey);
+    }
+
+    @Override
+    public String toString() {
+        return key + " => " + nextStepKey;
+    }
+
+    public static final class StepKey implements Writeable, ToXContentObject {
+        private final String phase;
+        private final String action;
+        private final String name;
+
+        public static final ParseField PHASE_FIELD = new ParseField("phase");
+        public static final ParseField ACTION_FIELD = new ParseField("action");
+        public static final ParseField NAME_FIELD = new ParseField("name");
+        private static final ConstructingObjectParser<StepKey, Void> PARSER =
+            new ConstructingObjectParser<>("stepkey", a -> new StepKey((String) a[0], (String) a[1], (String) a[2]));
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), PHASE_FIELD);
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), ACTION_FIELD);
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
+        }
+
+        public StepKey(String phase, String action, String name) {
+            this.phase = phase;
+            this.action = action;
+            this.name = name;
+        }
+
+        public StepKey(StreamInput in) throws IOException {
+            this.phase = in.readString();
+            this.action = in.readString();
+            this.name = in.readString();
+        }
+
+        public static StepKey parse(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(phase);
+            out.writeString(action);
+            out.writeString(name);
+        }
+
+        public String getPhase() {
+            return phase;
+        }
+
+        public String getAction() {
+            return action;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(phase, action, name);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            StepKey other = (StepKey) obj;
+            return Objects.equals(phase, other.phase) &&
+                    Objects.equals(action, other.action) &&
+                    Objects.equals(name, other.name);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(PHASE_FIELD.getPreferredName(), phase);
+            builder.field(ACTION_FIELD.getPreferredName(), action);
+            builder.field(NAME_FIELD.getPreferredName(), name);
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequest.java
new file mode 100644
index 0000000000000..3a2d458406b30
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+
+public class StopILMRequest extends AcknowledgedRequest<StopILMRequest> {
+
+    public StopILMRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return 75;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java
new file mode 100644
index 0000000000000..4ba1b4fd83c60
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStep.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+public class TerminalPolicyStep extends Step {
+    public static final String COMPLETED_PHASE = "completed";
+    public static final StepKey KEY = new StepKey(COMPLETED_PHASE, "completed", "completed");
+    public static final TerminalPolicyStep INSTANCE = new TerminalPolicyStep(KEY, null);
+
+    TerminalPolicyStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java
new file mode 100644
index 0000000000000..17c9eaf17c083
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Represents the lifecycle of an index from creation to deletion. A
+ * {@link TimeseriesLifecycleType} is made up of a set of {@link Phase}s which it will
+ * move through. Soon we will constrain the phases using some kinda of lifecycle
+ * type which will allow only particular {@link Phase}s to be defined, will
+ * dictate the order in which the {@link Phase}s are executed and will define
+ * which {@link LifecycleAction}s are allowed in each phase.
+ */
+public class TimeseriesLifecycleType implements LifecycleType {
+    public static final TimeseriesLifecycleType INSTANCE = new TimeseriesLifecycleType();
+
+    public static final String TYPE = "timeseries";
+    static final List<String> VALID_PHASES = Arrays.asList("hot", "warm", "cold", "delete");
+    static final List<String> ORDERED_VALID_HOT_ACTIONS = Collections.singletonList(RolloverAction.NAME);
+    static final List<String> ORDERED_VALID_WARM_ACTIONS = Arrays.asList(ReadOnlyAction.NAME, AllocateAction.NAME,
+        ShrinkAction.NAME, ForceMergeAction.NAME);
+    static final List<String> ORDERED_VALID_COLD_ACTIONS = Arrays.asList(AllocateAction.NAME);
+    static final List<String> ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(DeleteAction.NAME);
+    static final Set<String> VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS);
+    static final Set<String> VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS);
+    static final Set<String> VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS);
+    static final Set<String> VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS);
+    private static final Phase EMPTY_WARM_PHASE = new Phase("warm", TimeValue.ZERO,
+        Collections.singletonMap("readonly", ReadOnlyAction.INSTANCE));
+    private static Map<String, Set<String>> ALLOWED_ACTIONS = new HashMap<>();
+
+    static {
+        ALLOWED_ACTIONS.put("hot", VALID_HOT_ACTIONS);
+        ALLOWED_ACTIONS.put("warm", VALID_WARM_ACTIONS);
+        ALLOWED_ACTIONS.put("cold", VALID_COLD_ACTIONS);
+        ALLOWED_ACTIONS.put("delete", VALID_DELETE_ACTIONS);
+    }
+
+    private TimeseriesLifecycleType() {
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    public List<Phase> getOrderedPhases(Map<String, Phase> phases) {
+        List<Phase> orderedPhases = new ArrayList<>(VALID_PHASES.size());
+        for (String phaseName : VALID_PHASES) {
+            Phase phase = phases.get(phaseName);
+            if (phase != null) {
+                orderedPhases.add(phase);
+            }
+        }
+        return orderedPhases;
+    }
+
+    @Override
+    public String getNextPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        int index = VALID_PHASES.indexOf(currentPhaseName);
+        if (index < 0 && "new".equals(currentPhaseName) == false) {
+            throw new IllegalArgumentException("[" + currentPhaseName + "] is not a valid phase for lifecycle type [" + TYPE + "]");
+        } else {
+            // Find the next phase after `index` that exists in `phases` and return it
+            while (++index < VALID_PHASES.size()) {
+                String phaseName = VALID_PHASES.get(index);
+                if (phases.containsKey(phaseName)) {
+                    return phaseName;
+                }
+            }
+            // if we have exhausted VALID_PHASES and haven't found a matching
+            // phase in `phases` return null indicating there is no next phase
+            // available
+            return null;
+        }
+    }
+
+    public String getPreviousPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        if ("new".equals(currentPhaseName)) {
+            return null;
+        }
+        int index = VALID_PHASES.indexOf(currentPhaseName);
+        if (index < 0) {
+            throw new IllegalArgumentException("[" + currentPhaseName + "] is not a valid phase for lifecycle type [" + TYPE + "]");
+        } else {
+            // Find the previous phase before `index` that exists in `phases` and return it
+            while (--index >=0) {
+                String phaseName = VALID_PHASES.get(index);
+                if (phases.containsKey(phaseName)) {
+                    return phaseName;
+                }
+            }
+            // if we have exhausted VALID_PHASES and haven't found a matching
+            // phase in `phases` return null indicating there is no previous phase
+            // available
+            return null;
+        }
+    }
+
+    public List<LifecycleAction> getOrderedActions(Phase phase) {
+        Map<String, LifecycleAction> actions = phase.getActions();
+        switch (phase.getName()) {
+            case "hot":
+                return ORDERED_VALID_HOT_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
+                    .filter(Objects::nonNull).collect(Collectors.toList());
+            case "warm":
+                return ORDERED_VALID_WARM_ACTIONS.stream() .map(a -> actions.getOrDefault(a, null))
+                    .filter(Objects::nonNull).collect(Collectors.toList());
+            case "cold":
+                return ORDERED_VALID_COLD_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
+                    .filter(Objects::nonNull).collect(Collectors.toList());
+            case "delete":
+                return ORDERED_VALID_DELETE_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
+                    .filter(Objects::nonNull).collect(Collectors.toList());
+            default:
+                throw new IllegalArgumentException("lifecycle type[" + TYPE + "] does not support phase[" + phase.getName() + "]");
+        }
+    }
+
+    @Override
+    public String getNextActionName(String currentActionName, Phase phase) {
+        List<String> orderedActionNames;
+        switch (phase.getName()) {
+        case "hot":
+            orderedActionNames = ORDERED_VALID_HOT_ACTIONS;
+            break;
+        case "warm":
+            orderedActionNames = ORDERED_VALID_WARM_ACTIONS;
+            break;
+        case "cold":
+            orderedActionNames = ORDERED_VALID_COLD_ACTIONS;
+            break;
+        case "delete":
+            orderedActionNames = ORDERED_VALID_DELETE_ACTIONS;
+            break;
+        default:
+            throw new IllegalArgumentException("lifecycle type[" + TYPE + "] does not support phase[" + phase.getName() + "]");
+        }
+
+        int index = orderedActionNames.indexOf(currentActionName);
+        if (index < 0) {
+            throw new IllegalArgumentException("[" + currentActionName + "] is not a valid action for phase [" + phase.getName()
+                    + "] in lifecycle type [" + TYPE + "]");
+        } else {
+            // Find the next action after `index` that exists in the phase and return it
+            while (++index < orderedActionNames.size()) {
+                String actionName = orderedActionNames.get(index);
+                if (phase.getActions().containsKey(actionName)) {
+                    return actionName;
+                }
+            }
+            // if we have exhausted `validActions` and haven't found a matching
+            // action in the Phase return null indicating there is no next
+            // action available
+            return null;
+        }
+    }
+
+    @Override
+    public void validate(Collection<Phase> phases) {
+        phases.forEach(phase -> {
+            if (ALLOWED_ACTIONS.containsKey(phase.getName()) == false) {
+                throw new IllegalArgumentException("Timeseries lifecycle does not support phase [" + phase.getName() + "]");
+            }
+            phase.getActions().forEach((actionName, action) -> {
+                if (ALLOWED_ACTIONS.get(phase.getName()).contains(actionName) == false) {
+                    throw new IllegalArgumentException("invalid action [" + actionName + "] " +
+                        "defined in phase [" + phase.getName() +"]");
+                }
+            });
+        });
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java
new file mode 100644
index 0000000000000..704d122f571a6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStep.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.index.Index;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
+public class UpdateRolloverLifecycleDateStep extends ClusterStateActionStep {
+    public static final String NAME = "update-rollover-lifecycle-date";
+
+    public UpdateRolloverLifecycleDateStep(StepKey key, StepKey nextStepKey) {
+        super(key, nextStepKey);
+    }
+
+    @Override
+    public ClusterState performAction(Index index, ClusterState currentState) {
+        IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
+        // find the newly created index from the rollover and fetch its index.creation_date
+        String rolloverAlias = RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.get(indexMetaData.getSettings());
+        if (Strings.isNullOrEmpty(rolloverAlias)) {
+            throw new IllegalStateException("setting [" + RolloverAction.LIFECYCLE_ROLLOVER_ALIAS
+                + "] is not set on index [" + indexMetaData.getIndex().getName() + "]");
+        }
+        RolloverInfo rolloverInfo = indexMetaData.getRolloverInfos().get(rolloverAlias);
+        if (rolloverInfo == null) {
+            throw new IllegalStateException("no rollover info found for [" + indexMetaData.getIndex().getName() + "], either the index " +
+                "has not yet rolled over or a subsequent index was created outside of Index Lifecycle Management");
+        }
+
+        LifecycleExecutionState.Builder newLifecycleState = LifecycleExecutionState
+            .builder(LifecycleExecutionState.fromIndexMetadata(indexMetaData));
+        newLifecycleState.setIndexCreationDate(rolloverInfo.getTime());
+
+        IndexMetaData.Builder newIndexMetadata = IndexMetaData.builder(indexMetaData);
+        newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap());
+        return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData())
+            .put(newIndexMetadata)).build();
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj != null && getClass() == obj.getClass() && super.equals(obj);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStep.java
new file mode 100644
index 0000000000000..5602f6aa3f499
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStep.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.Objects;
+
+public class UpdateSettingsStep extends AsyncActionStep {
+    public static final String NAME = "update-settings";
+
+    private final Settings settings;
+
+    public UpdateSettingsStep(StepKey key, StepKey nextStepKey, Client client, Settings settings) {
+        super(key, nextStepKey, client);
+        this.settings = settings;
+    }
+
+    @Override
+    public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexMetaData.getIndex().getName()).settings(settings);
+        getClient().admin().indices().updateSettings(updateSettingsRequest,
+                ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
+    }
+
+    public Settings getSettings() {
+        return settings;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), settings);
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        UpdateSettingsStep other = (UpdateSettingsStep) obj;
+        return super.equals(obj) &&
+                Objects.equals(settings, other.settings);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java
new file mode 100644
index 0000000000000..2a2c65a2b18c8
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteLifecycleAction
+        extends Action<DeleteLifecycleAction.Request, DeleteLifecycleAction.Response, DeleteLifecycleActionRequestBuilder> {
+    public static final DeleteLifecycleAction INSTANCE = new DeleteLifecycleAction();
+    public static final String NAME = "cluster:admin/ilm/delete";
+
+    protected DeleteLifecycleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+
+        public Response() {
+        }
+
+        public Response(boolean acknowledged) {
+            super(acknowledged);
+        }
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> {
+
+        public static final ParseField POLICY_FIELD = new ParseField("policy");
+
+        private String policyName;
+
+        public Request(String policyName) {
+            this.policyName = policyName;
+        }
+
+        public Request() {
+        }
+
+        public String getPolicyName() {
+            return policyName;
+        }
+
+        public void setPolicyName(final String policyName) {
+            this.policyName = policyName;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            policyName = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(policyName);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(policyName);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.equals(policyName, other.policyName);
+        }
+
+    }
+
+    @Override
+    public DeleteLifecycleActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new DeleteLifecycleActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleActionRequestBuilder.java
new file mode 100644
index 0000000000000..294adeb6d4303
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleActionRequestBuilder.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class DeleteLifecycleActionRequestBuilder
+        extends ActionRequestBuilder<DeleteLifecycleAction.Request, DeleteLifecycleAction.Response, DeleteLifecycleActionRequestBuilder> {
+
+    public DeleteLifecycleActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<DeleteLifecycleAction.Request, DeleteLifecycleAction.Response, DeleteLifecycleActionRequestBuilder> action) {
+        super(client, action, new DeleteLifecycleAction.Request());
+    }
+
+    public DeleteLifecycleActionRequestBuilder setPolicyName(final String policyName) {
+        request.setPolicyName(policyName);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java
new file mode 100644
index 0000000000000..8097f7c0f979d
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class ExplainLifecycleAction
+        extends Action<ExplainLifecycleRequest, ExplainLifecycleResponse, ExplainLifecycleActionRequestBuilder> {
+    public static final ExplainLifecycleAction INSTANCE = new ExplainLifecycleAction();
+    public static final String NAME = "indices:admin/ilm/explain";
+
+    protected ExplainLifecycleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public ExplainLifecycleResponse newResponse() {
+        return new ExplainLifecycleResponse();
+    }
+
+    @Override
+    public ExplainLifecycleActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new ExplainLifecycleActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleActionRequestBuilder.java
new file mode 100644
index 0000000000000..f4a92462d977e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleActionRequestBuilder.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.master.info.ClusterInfoRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
+
+public class ExplainLifecycleActionRequestBuilder
+        extends ClusterInfoRequestBuilder<ExplainLifecycleRequest, ExplainLifecycleResponse, ExplainLifecycleActionRequestBuilder> {
+
+    public ExplainLifecycleActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<ExplainLifecycleRequest, ExplainLifecycleResponse, ExplainLifecycleActionRequestBuilder> action) {
+        super(client, action, new ExplainLifecycleRequest());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java
new file mode 100644
index 0000000000000..f8c47905c8e63
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java
@@ -0,0 +1,221 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class GetLifecycleAction extends Action<GetLifecycleAction.Request, GetLifecycleAction.Response, GetLifecycleActionRequestBuilder> {
+    public static final GetLifecycleAction INSTANCE = new GetLifecycleAction();
+    public static final String NAME = "cluster:admin/ilm/get";
+
+    protected GetLifecycleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private List<LifecyclePolicyResponseItem> policies;
+
+        public Response() {
+        }
+
+        public Response(List<LifecyclePolicyResponseItem> policies) {
+            this.policies = policies;
+        }
+
+        public List<LifecyclePolicyResponseItem> getPolicies() {
+            return policies;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            for (LifecyclePolicyResponseItem item : policies) {
+                builder.startObject(item.getLifecyclePolicy().getName());
+                builder.field("version", item.getVersion());
+                builder.field("modified_date", item.getModifiedDate());
+                builder.field("policy", item.getLifecyclePolicy());
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            this.policies = in.readList(LifecyclePolicyResponseItem::new);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeList(policies);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(policies);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(policies, other.policies);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
+        }
+
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> {
+        private String[] policyNames;
+
+        public Request(String... policyNames) {
+            if (policyNames == null) {
+                throw new IllegalArgumentException("ids cannot be null");
+            }
+            this.policyNames = policyNames;
+        }
+
+        public Request() {
+            policyNames = Strings.EMPTY_ARRAY;
+        }
+
+        public String[] getPolicyNames() {
+            return policyNames;
+        }
+
+        public void setPolicyNames(final String[] policyNames) {
+            this.policyNames = policyNames;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            policyNames = in.readStringArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(policyNames);
+        }
+
+        @Override
+        public int hashCode() {
+            return Arrays.hashCode(policyNames);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Arrays.equals(policyNames, other.policyNames);
+        }
+
+    }
+
+    @Override
+    public GetLifecycleActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new GetLifecycleActionRequestBuilder(client, INSTANCE);
+    }    
+
+    public static class LifecyclePolicyResponseItem implements Writeable {
+        private final LifecyclePolicy lifecyclePolicy;
+        private final long version;
+        private final String modifiedDate;
+
+        public LifecyclePolicyResponseItem(LifecyclePolicy lifecyclePolicy, long version, String modifiedDate) {
+            this.lifecyclePolicy = lifecyclePolicy;
+            this.version = version;
+            this.modifiedDate = modifiedDate;
+        }
+
+        LifecyclePolicyResponseItem(StreamInput in) throws IOException {
+            this.lifecyclePolicy = new LifecyclePolicy(in);
+            this.version = in.readVLong();
+            this.modifiedDate = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            lifecyclePolicy.writeTo(out);
+            out.writeVLong(version);
+            out.writeString(modifiedDate);
+        }
+
+        public LifecyclePolicy getLifecyclePolicy() {
+            return lifecyclePolicy;
+        }
+
+        public long getVersion() {
+            return version;
+        }
+
+        public String getModifiedDate() {
+            return modifiedDate;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(lifecyclePolicy, version, modifiedDate);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            LifecyclePolicyResponseItem other = (LifecyclePolicyResponseItem) obj;
+            return Objects.equals(lifecyclePolicy, other.lifecyclePolicy) &&
+                Objects.equals(version, other.version) &&
+                Objects.equals(modifiedDate, other.modifiedDate);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleActionRequestBuilder.java
new file mode 100644
index 0000000000000..ede22a6746a20
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleActionRequestBuilder.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class GetLifecycleActionRequestBuilder
+        extends ActionRequestBuilder<GetLifecycleAction.Request, GetLifecycleAction.Response, GetLifecycleActionRequestBuilder> {
+
+    public GetLifecycleActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<GetLifecycleAction.Request, GetLifecycleAction.Response, GetLifecycleActionRequestBuilder> action) {
+        super(client, action, new GetLifecycleAction.Request());
+    }
+
+    public GetLifecycleActionRequestBuilder setPolicyNames(final String[] policyNames) {
+        request.setPolicyNames(policyNames);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java
new file mode 100644
index 0000000000000..988342022731a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class GetStatusAction extends Action<GetStatusAction.Request, GetStatusAction.Response, GetStatusActionRequestBuilder> {
+    public static final GetStatusAction INSTANCE = new GetStatusAction();
+    public static final String NAME = "cluster:admin/ilm/operation_mode/get";
+
+    protected GetStatusAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private OperationMode mode;
+
+        public Response() {
+        }
+
+        public Response(OperationMode mode) {
+            this.mode = mode;
+        }
+
+        public OperationMode getMode() {
+            return mode;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("operation_mode", mode);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            mode = in.readEnum(OperationMode.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeEnum(mode);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(mode);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(mode, other.mode);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
+        }
+
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> {
+
+        public Request() {
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+        }
+    }
+
+    @Override
+    public GetStatusActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new GetStatusActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusActionRequestBuilder.java
new file mode 100644
index 0000000000000..8aa09cb7dedb2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusActionRequestBuilder.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class GetStatusActionRequestBuilder
+        extends ActionRequestBuilder<GetStatusAction.Request, GetStatusAction.Response, GetStatusActionRequestBuilder> {
+
+    public GetStatusActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<GetStatusAction.Request, GetStatusAction.Response, GetStatusActionRequestBuilder> action) {
+        super(client, action, new GetStatusAction.Request());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java
new file mode 100644
index 0000000000000..a122cc85d7699
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class MoveToStepAction extends Action<MoveToStepAction.Request, MoveToStepAction.Response, MoveToStepActionRequestBuilder> {
+    public static final MoveToStepAction INSTANCE = new MoveToStepAction();
+    public static final String NAME = "cluster:admin/ilm/_move/post";
+
+    protected MoveToStepAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+
+        public Response() {
+        }
+
+        public Response(boolean acknowledged) {
+            super(acknowledged);
+        }
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+        static final ParseField CURRENT_KEY_FIELD = new ParseField("current_step");
+        static final ParseField NEXT_KEY_FIELD = new ParseField("next_step");
+        private static final ConstructingObjectParser<Request, String> PARSER =
+            new ConstructingObjectParser<>("move_to_step_request", false,
+                (a, index) -> {
+                    StepKey currentStepKey = (StepKey) a[0];
+                    StepKey nextStepKey = (StepKey) a[1];
+                    return new Request(index, currentStepKey, nextStepKey);
+                });
+        static {
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, name) -> StepKey.parse(p), CURRENT_KEY_FIELD);
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, name) -> StepKey.parse(p), NEXT_KEY_FIELD);
+        }
+
+        private String index;
+        private StepKey currentStepKey;
+        private StepKey nextStepKey;
+
+        public Request(String index, StepKey currentStepKey, StepKey nextStepKey) {
+            this.index = index;
+            this.currentStepKey = currentStepKey;
+            this.nextStepKey = nextStepKey;
+        }
+
+        public Request() {
+        }
+
+        public String getIndex() {
+            return index;
+        }
+
+        public void setIndex(final String index) {
+            this.index = index;
+        }
+
+        public StepKey getCurrentStepKey() {
+            return currentStepKey;
+        }
+
+        public void setCurrentStepKey(final StepKey currentStepKey) {
+            this.currentStepKey = currentStepKey;
+        }
+
+        public StepKey getNextStepKey() {
+            return nextStepKey;
+        }
+
+        public void setNextStepKey(final StepKey nextStepKey) {
+            this.nextStepKey = nextStepKey;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public static Request parseRequest(String name, XContentParser parser) {
+            return PARSER.apply(parser, name);
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            this.index = in.readString();
+            this.currentStepKey = new StepKey(in);
+            this.nextStepKey = new StepKey(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(index);
+            currentStepKey.writeTo(out);
+            nextStepKey.writeTo(out);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(index, currentStepKey, nextStepKey);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.equals(index, other.index) && Objects.equals(currentStepKey, other.currentStepKey)
+                && Objects.equals(nextStepKey, other.nextStepKey);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject()
+                .field(CURRENT_KEY_FIELD.getPreferredName(), currentStepKey)
+                .field(NEXT_KEY_FIELD.getPreferredName(), nextStepKey)
+                .endObject();
+        }
+    }
+
+    @Override
+    public MoveToStepActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new MoveToStepActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepActionRequestBuilder.java
new file mode 100644
index 0000000000000..c6b3506b9c908
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepActionRequestBuilder.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+
+public class MoveToStepActionRequestBuilder
+        extends ActionRequestBuilder<MoveToStepAction.Request, MoveToStepAction.Response, MoveToStepActionRequestBuilder> {
+
+    public MoveToStepActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<MoveToStepAction.Request, MoveToStepAction.Response, MoveToStepActionRequestBuilder> action) {
+        super(client, action, new MoveToStepAction.Request());
+    }
+
+    public MoveToStepActionRequestBuilder setIndex(final String index) {
+        request.setIndex(index);
+        return this;
+    }
+
+    public MoveToStepActionRequestBuilder setCurrentStepKey(final Step.StepKey currentStepKey) {
+        request.setCurrentStepKey(currentStepKey);
+        return this;
+    }
+
+    public MoveToStepActionRequestBuilder setNextStepKey(final Step.StepKey nextStepKey) {
+        request.setNextStepKey(nextStepKey);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java
new file mode 100644
index 0000000000000..527ea14140ad3
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutLifecycleAction extends Action<PutLifecycleAction.Request, PutLifecycleAction.Response, PutLifecycleActionRequestBuilder> {
+    public static final PutLifecycleAction INSTANCE = new PutLifecycleAction();
+    public static final String NAME = "cluster:admin/ilm/put";
+
+    protected PutLifecycleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+
+        public Response() {
+        }
+
+        public Response(boolean acknowledged) {
+            super(acknowledged);
+        }
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+
+        public static final ParseField POLICY_FIELD = new ParseField("policy");
+        private static final ConstructingObjectParser<Request, String> PARSER =
+            new ConstructingObjectParser<>("put_lifecycle_request", a -> new Request((LifecyclePolicy) a[0]));
+        static {
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), LifecyclePolicy::parse, POLICY_FIELD);
+        }
+
+        private LifecyclePolicy policy;
+
+        public Request(LifecyclePolicy policy) {
+            this.policy = policy;
+        }
+
+        public Request() {
+        }
+
+        public LifecyclePolicy getPolicy() {
+            return policy;
+        }
+
+        public void setPolicy(final LifecyclePolicy policy) {
+            this.policy = policy;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        public static Request parseRequest(String name, XContentParser parser) {
+            return PARSER.apply(parser, name);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(POLICY_FIELD.getPreferredName(), policy);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            policy = new LifecyclePolicy(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            policy.writeTo(out);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(policy);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.equals(policy, other.policy);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
+        }
+
+    }
+
+    @Override
+    public PutLifecycleActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new PutLifecycleActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleActionRequestBuilder.java
new file mode 100644
index 0000000000000..5da8fbb8afdc1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleActionRequestBuilder.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+
+public class PutLifecycleActionRequestBuilder
+        extends ActionRequestBuilder<PutLifecycleAction.Request, PutLifecycleAction.Response, PutLifecycleActionRequestBuilder> {
+
+    public PutLifecycleActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<PutLifecycleAction.Request, PutLifecycleAction.Response, PutLifecycleActionRequestBuilder> action) {
+        super(client, action, new PutLifecycleAction.Request());
+    }
+
+    public PutLifecycleActionRequestBuilder setPolicy(LifecyclePolicy policy) {
+        request.setPolicy(policy);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java
new file mode 100644
index 0000000000000..7058fa89c3e28
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class RemoveIndexLifecyclePolicyAction extends
+        Action<RemoveIndexLifecyclePolicyAction.Request, 
+               RemoveIndexLifecyclePolicyAction.Response, 
+               RemovePolicyForIndexActionRequestBuilder> {
+    public static final RemoveIndexLifecyclePolicyAction INSTANCE = new RemoveIndexLifecyclePolicyAction();
+    public static final String NAME = "indices:admin/ilm/remove_policy";
+
+    protected RemoveIndexLifecyclePolicyAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RemoveIndexLifecyclePolicyAction.Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        public static final ParseField HAS_FAILURES_FIELD = new ParseField("has_failures");
+        public static final ParseField FAILED_INDEXES_FIELD = new ParseField("failed_indexes");
+        @SuppressWarnings("unchecked")
+        public static final ConstructingObjectParser<Response, Void> PARSER = new ConstructingObjectParser<>(
+                "change_policy_for_index_response", a -> new Response((List<String>) a[0]));
+        static {
+            PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), FAILED_INDEXES_FIELD);
+            // Needs to be declared but not used in constructing the response object
+            PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), HAS_FAILURES_FIELD);
+        }
+
+        private List<String> failedIndexes;
+
+        public Response() {
+        }
+
+        public Response(List<String> failedIndexes) {
+            if (failedIndexes == null) {
+                throw new IllegalArgumentException(FAILED_INDEXES_FIELD.getPreferredName() + " cannot be null");
+            }
+            this.failedIndexes = failedIndexes;
+        }
+
+        public List<String> getFailedIndexes() {
+            return failedIndexes;
+        }
+
+        public boolean hasFailures() {
+            return failedIndexes.isEmpty() == false;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(HAS_FAILURES_FIELD.getPreferredName(), hasFailures());
+            builder.field(FAILED_INDEXES_FIELD.getPreferredName(), failedIndexes);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            failedIndexes = in.readList(StreamInput::readString);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringList(failedIndexes);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(failedIndexes);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(failedIndexes, other.failedIndexes);
+        }
+
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
+
+        private String[] indices;
+        private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+
+        public Request() {
+        }
+
+        public Request(String... indices) {
+            if (indices == null) {
+                throw new IllegalArgumentException("indices cannot be null");
+            }
+            this.indices = indices;
+        }
+
+        @Override
+        public Request indices(String... indices) {
+            this.indices = indices;
+            return this;
+        }
+
+        @Override
+        public String[] indices() {
+            return indices;
+        }
+
+        public void indicesOptions(IndicesOptions indicesOptions) {
+            this.indicesOptions = indicesOptions;
+        }
+
+        public IndicesOptions indicesOptions() {
+            return indicesOptions;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            indices = in.readStringArray();
+            indicesOptions = IndicesOptions.readIndicesOptions(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(indices);
+            indicesOptions.writeIndicesOptions(out);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(Arrays.hashCode(indices), indicesOptions);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.deepEquals(indices, other.indices) &&
+                    Objects.equals(indicesOptions, other.indicesOptions);
+        }
+
+    }
+
+    @Override
+    public RemovePolicyForIndexActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new RemovePolicyForIndexActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemovePolicyForIndexActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemovePolicyForIndexActionRequestBuilder.java
new file mode 100644
index 0000000000000..94e63cbe46b34
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemovePolicyForIndexActionRequestBuilder.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class RemovePolicyForIndexActionRequestBuilder
+        extends ActionRequestBuilder<
+        RemoveIndexLifecyclePolicyAction.Request,
+        RemoveIndexLifecyclePolicyAction.Response,
+        RemovePolicyForIndexActionRequestBuilder> {
+
+    public RemovePolicyForIndexActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<
+                    RemoveIndexLifecyclePolicyAction.Request,
+                    RemoveIndexLifecyclePolicyAction.Response,
+                    RemovePolicyForIndexActionRequestBuilder> action) {
+        super(client, action, new RemoveIndexLifecyclePolicyAction.Request());
+    }
+
+    public RemovePolicyForIndexActionRequestBuilder setIndices(final String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public RemovePolicyForIndexActionRequestBuilder setIndicesOptions(final IndicesOptions indicesOptions) {
+        request.indicesOptions(indicesOptions);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java
new file mode 100644
index 0000000000000..29585b4efa579
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class RetryAction extends Action<RetryAction.Request, RetryAction.Response, RetryActionRequestBuilder> {
+    public static final RetryAction INSTANCE = new RetryAction();
+    public static final String NAME = "indices:admin/ilm/retry";
+
+    protected RetryAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+
+        public Response() {
+        }
+
+        public Response(boolean acknowledged) {
+            super(acknowledged);
+        }
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
+        private String[] indices = Strings.EMPTY_ARRAY;
+        private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+
+        public Request(String... indices) {
+            this.indices = indices;
+        }
+
+        public Request() {
+        }
+
+        @Override
+        public Request indices(String... indices) {
+            this.indices = indices;
+            return this;
+        }
+
+        @Override
+        public String[] indices() {
+            return indices;
+        }
+
+        @Override
+        public IndicesOptions indicesOptions() {
+            return indicesOptions;
+        }
+
+        public Request indicesOptions(IndicesOptions indicesOptions) {
+            this.indicesOptions = indicesOptions;
+            return this;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            this.indices = in.readStringArray();
+            this.indicesOptions = IndicesOptions.readIndicesOptions(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(indices);
+            indicesOptions.writeIndicesOptions(out);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(Arrays.hashCode(indices), indicesOptions);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (obj.getClass() != getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.deepEquals(indices, other.indices)
+                && Objects.equals(indicesOptions, other.indicesOptions);
+        }
+
+    }
+
+    @Override
+    public RetryActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new RetryActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryActionRequestBuilder.java
new file mode 100644
index 0000000000000..cfdbced22f160
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryActionRequestBuilder.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class RetryActionRequestBuilder extends ActionRequestBuilder<RetryAction.Request, RetryAction.Response, RetryActionRequestBuilder> {
+
+    public RetryActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<RetryAction.Request, RetryAction.Response, RetryActionRequestBuilder> action) {
+        super(client, action, new RetryAction.Request());
+    }
+
+    public RetryActionRequestBuilder setIndices(final String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public RetryActionRequestBuilder setIndicesOptions(final IndicesOptions indicesOptions) {
+        request.indicesOptions(indicesOptions);
+        return this;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java
new file mode 100644
index 0000000000000..c72344b36cfaa
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
+
+public class StartILMAction extends Action<StartILMRequest, AcknowledgedResponse, StartILMActionRequestBuilder> {
+    public static final StartILMAction INSTANCE = new StartILMAction();
+    public static final String NAME = "cluster:admin/ilm/start";
+
+    protected StartILMAction() {
+        super(NAME);
+    }
+
+    @Override
+    public AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
+    }
+
+    @Override
+    public StartILMActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new StartILMActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMActionRequestBuilder.java
new file mode 100644
index 0000000000000..fbf336295853f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMActionRequestBuilder.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
+
+public class StartILMActionRequestBuilder
+        extends ActionRequestBuilder<StartILMRequest, AcknowledgedResponse, StartILMActionRequestBuilder> {
+
+    public StartILMActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<StartILMRequest, AcknowledgedResponse, StartILMActionRequestBuilder> action) {
+        super(client, action, new StartILMRequest());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java
new file mode 100644
index 0000000000000..854da203e6c14
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
+
+public class StopILMAction extends Action<StopILMRequest, AcknowledgedResponse, StopILMActionRequestBuilder> {
+    public static final StopILMAction INSTANCE = new StopILMAction();
+    public static final String NAME = "cluster:admin/ilm/stop";
+
+    protected StopILMAction() {
+        super(NAME);
+    }
+
+    @Override
+    public AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
+    }
+
+    @Override
+    public StopILMActionRequestBuilder newRequestBuilder(final ElasticsearchClient client) {
+        return new StopILMActionRequestBuilder(client, INSTANCE);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMActionRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMActionRequestBuilder.java
new file mode 100644
index 0000000000000..8b6cdb4dee67e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMActionRequestBuilder.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
+
+public class StopILMActionRequestBuilder extends ActionRequestBuilder<StopILMRequest, AcknowledgedResponse, StopILMActionRequestBuilder> {
+
+    public StopILMActionRequestBuilder(
+            final ElasticsearchClient client,
+            final Action<StopILMRequest, AcknowledgedResponse, StopILMActionRequestBuilder> action) {
+        super(client, action, new StopILMRequest());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java
new file mode 100644
index 0000000000000..5e81d2e2aa7e5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.client;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
+
+/**
+ * A wrapper to elasticsearch clients that exposes all ILM related APIs
+ */
+public class ILMClient {
+
+    private ElasticsearchClient client;
+
+    public ILMClient(ElasticsearchClient client) {
+        this.client = client;
+    }
+
+    /**
+     * Create or modify a lifecycle policy definition
+     */
+    public void putLifecyclePolicy(PutLifecycleAction.Request request, ActionListener<PutLifecycleAction.Response> listener) {
+        client.execute(PutLifecycleAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Create or modify a lifecycle policy definition
+     */
+    public ActionFuture<PutLifecycleAction.Response> putLifecyclePolicy(PutLifecycleAction.Request request) {
+        return client.execute(PutLifecycleAction.INSTANCE, request);
+    }
+
+    /**
+     * Get a lifecycle policy definition
+     */
+    public void getLifecyclePolicy(GetLifecycleAction.Request request, ActionListener<GetLifecycleAction.Response> listener) {
+        client.execute(GetLifecycleAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Get a lifecycle policy definition
+     */
+    public ActionFuture<GetLifecycleAction.Response> getLifecyclePolicy(GetLifecycleAction.Request request) {
+        return client.execute(GetLifecycleAction.INSTANCE, request);
+    }
+
+    /**
+     * Delete a lifecycle policy definition
+     */
+    public void deleteLifecyclePolicy(DeleteLifecycleAction.Request request, ActionListener<DeleteLifecycleAction.Response> listener) {
+        client.execute(DeleteLifecycleAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Delete a lifecycle policy definition
+     */
+    public ActionFuture<DeleteLifecycleAction.Response> deleteLifecyclePolicy(DeleteLifecycleAction.Request request) {
+        return client.execute(DeleteLifecycleAction.INSTANCE, request);
+    }
+
+    /**
+     * Explain the current lifecycle state for an index
+     */
+    public void explainLifecycle(ExplainLifecycleRequest request, ActionListener<ExplainLifecycleResponse> listener) {
+        client.execute(ExplainLifecycleAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Explain the current lifecycle state for an index
+     */
+    public ActionFuture<ExplainLifecycleResponse> explainLifecycle(ExplainLifecycleRequest request) {
+        return client.execute(ExplainLifecycleAction.INSTANCE, request);
+    }
+
+    /**
+     * Returns the current status of the ILM plugin
+     */
+    public void getStatus(GetStatusAction.Request request, ActionListener<GetStatusAction.Response> listener) {
+        client.execute(GetStatusAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Returns the current status of the ILM plugin
+     */
+    public ActionFuture<GetStatusAction.Response> getStatus(GetStatusAction.Request request) {
+        return client.execute(GetStatusAction.INSTANCE, request);
+    }
+
+    /**
+     * Removes index lifecycle management from an index
+     */
+    public void removeIndexLifecyclePolicy(RemoveIndexLifecyclePolicyAction.Request request,
+            ActionListener<RemoveIndexLifecyclePolicyAction.Response> listener) {
+        client.execute(RemoveIndexLifecyclePolicyAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Removes index lifecycle management from an index
+     */
+    public ActionFuture<RemoveIndexLifecyclePolicyAction.Response> removeIndexLifecyclePolicy(
+            RemoveIndexLifecyclePolicyAction.Request request) {
+        return client.execute(RemoveIndexLifecyclePolicyAction.INSTANCE, request);
+    }
+
+    /**
+     * Retries the policy for an index which is currently in ERROR
+     */
+    public void retryPolicy(RetryAction.Request request, ActionListener<RetryAction.Response> listener) {
+        client.execute(RetryAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Removes index lifecycle management from an index
+     */
+    public ActionFuture<RetryAction.Response> retryPolicy(RetryAction.Request request) {
+        return client.execute(RetryAction.INSTANCE, request);
+    }
+
+    /**
+     * Starts the ILM plugin
+     */
+    public void startILM(StartILMRequest request, ActionListener<AcknowledgedResponse> listener) {
+        client.execute(StartILMAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Starts the ILM plugin
+     */
+    public ActionFuture<AcknowledgedResponse> startILM(StartILMRequest request) {
+        return client.execute(StartILMAction.INSTANCE, request);
+    }
+
+    /**
+     * Stops the ILM plugin
+     */
+    public void stopILM(StopILMRequest request, ActionListener<AcknowledgedResponse> listener) {
+        client.execute(StopILMAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Stops the ILM plugin
+     */
+    public ActionFuture<AcknowledgedResponse> stopILM(StopILMRequest request) {
+        return client.execute(StopILMAction.INSTANCE, request);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
index 48ec7c2bfda4d..1b4a26efa5234 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
@@ -445,9 +445,9 @@ private TimeValue defaultFrequencyTarget(TimeValue bucketSpan) {
 
     public static class Builder {
 
+        public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
         private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
         private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
-        private static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
 
         private String id;
         private String jobId;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/ExtractorUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/ExtractorUtils.java
index 6d9312654facb..f0ba07ad15c68 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/ExtractorUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/ExtractorUtils.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.datafeed.extractor;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.rounding.DateTimeUnit;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -34,7 +34,7 @@
  */
 public final class ExtractorUtils {
 
-    private static final Logger LOGGER = Loggers.getLogger(ExtractorUtils.class);
+    private static final Logger LOGGER = LogManager.getLogger(ExtractorUtils.class);
     private static final String EPOCH_MILLIS = "epoch_millis";
 
     private ExtractorUtils() {}
@@ -139,7 +139,7 @@ private static long validateAndGetDateHistogramInterval(DateHistogramAggregation
         }
     }
 
-    static long validateAndGetCalendarInterval(String calendarInterval) {
+    public static long validateAndGetCalendarInterval(String calendarInterval) {
         TimeValue interval;
         DateTimeUnit dateTimeUnit = DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(calendarInterval);
         if (dateTimeUnit != null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
index b4d3d6efb7d47..18dfef4b7f4c4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
@@ -30,7 +30,11 @@ public SearchResponse newResponse() {
         return new SearchResponse();
     }
 
-    static class RequestBuilder extends ActionRequestBuilder<SearchRequest, SearchResponse, RequestBuilder> {
+    public static class RequestBuilder extends ActionRequestBuilder<SearchRequest, SearchResponse, RequestBuilder> {
+        public RequestBuilder(ElasticsearchClient client, SearchRequest searchRequest) {
+            super(client, INSTANCE, searchRequest);
+        }
+
         RequestBuilder(ElasticsearchClient client) {
             super(client, INSTANCE, new SearchRequest());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
index ea7a37b205431..f0e2f2b7e6217 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.authz.permission;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.ElasticsearchSecurityException;
@@ -14,7 +15,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
@@ -67,7 +67,7 @@ static Predicate<String> indexMatcher(List<String> indices) {
         try {
             return Automatons.predicate(indices);
         } catch (TooComplexToDeterminizeException e) {
-            Loggers.getLogger(IndicesPermission.class).debug("Index pattern automaton [{}] is too complex", indices);
+            LogManager.getLogger(IndicesPermission.class).debug("Index pattern automaton [{}] is too complex", indices);
             String description = Strings.collectionToCommaDelimitedString(indices);
             if (description.length() > 80) {
                 description = Strings.cleanTruncate(description, 80) + "...";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
index 2217513c03fe7..4f8d7372b0cc1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.core.ssl;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
@@ -35,8 +34,7 @@ public class SSLConfigurationReloader extends AbstractComponent {
     private final ResourceWatcherService resourceWatcherService;
     private final SSLService sslService;
 
-    public SSLConfigurationReloader(Settings settings, Environment env, SSLService sslService, ResourceWatcherService resourceWatcher) {
-        super(settings);
+    public SSLConfigurationReloader(Environment env, SSLService sslService, ResourceWatcherService resourceWatcher) {
         this.environment = env;
         this.resourceWatcherService = resourceWatcher;
         this.sslService = sslService;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
index 08513ce7412a4..00a0193e4604e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
@@ -60,6 +60,8 @@
  */
 public class SSLService extends AbstractComponent {
 
+    private final Settings settings;
+
     /**
      * This is a mapping from "context name" (in general use, the name of a setting key)
      * to a configuration.
@@ -86,7 +88,7 @@ public class SSLService extends AbstractComponent {
      * for use later
      */
     public SSLService(Settings settings, Environment environment) {
-        super(settings);
+        this.settings = settings;
         this.env = environment;
         this.globalSSLConfiguration = new SSLConfiguration(settings.getByPrefix(XPackSettings.GLOBAL_SSL_PREFIX));
         this.sslConfigurations = new HashMap<>();
@@ -95,7 +97,7 @@ public SSLService(Settings settings, Environment environment) {
 
     private SSLService(Settings settings, Environment environment, SSLConfiguration globalSSLConfiguration,
                        Map<String, SSLConfiguration> sslConfigurations, Map<SSLConfiguration, SSLContextHolder> sslContexts) {
-        super(settings);
+        this.settings = settings;
         this.env = environment;
         this.globalSSLConfiguration = globalSSLConfiguration;
         this.sslConfigurations = sslConfigurations;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
index a25e79ffdf66f..212c6d9c1c000 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
@@ -68,7 +68,6 @@ public class CryptoService extends AbstractComponent {
     private final SecretKey encryptionKey;
 
     public CryptoService(Settings settings) throws IOException {
-        super(settings);
         this.encryptionAlgorithm = ENCRYPTION_ALGO_SETTING.get(settings);
         final int keyLength = ENCRYPTION_KEY_LENGTH_SETTING.get(settings);
         this.ivLength = keyLength / 8;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java
index 059718973201c..e0724795c297c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java
@@ -12,11 +12,12 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.core.watcher.common.xcontent.XContentUtils;
+import org.elasticsearch.common.xcontent.XContentUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcStreamableXContentTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcStreamableXContentTestCase.java
new file mode 100644
index 0000000000000..d81fd5aed79ba
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcStreamableXContentTestCase.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.protocol;
+
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public abstract class AbstractHlrcStreamableXContentTestCase<T extends ToXContent & Streamable, H>
+        extends AbstractStreamableXContentTestCase<T> {
+
+    /**
+     * Generic test that creates new instance of HLRC request/response from the test instance and checks
+     * both for equality and asserts equality on the two queries.
+     */
+    public final void testHlrcFromXContent() throws IOException {
+        xContentTester(this::createParser, this::createTestInstance, getToXContentParams(),
+            p -> convertHlrcToInternal(doHlrcParseInstance(p)))
+            .numberOfTestRuns(NUMBER_OF_TEST_RUNS)
+            .supportsUnknownFields(supportsUnknownFields())
+            .shuffleFieldsExceptions(getShuffleFieldsExceptions())
+            .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter())
+            .assertEqualsConsumer(this::assertEqualInstances)
+            .assertToXContentEquivalence(true)
+            .test();
+    }
+
+    /**
+     * Parses to a new HLRC instance using the provided {@link XContentParser}
+     */
+    public abstract H doHlrcParseInstance(XContentParser parser) throws IOException;
+
+    /**
+     * Converts a HLRC instance to a XPack instance
+     */
+    public abstract T convertHlrcToInternal(H instance);
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcXContentTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcXContentTestCase.java
new file mode 100644
index 0000000000000..d6d8f9afe3659
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcXContentTestCase.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.protocol;
+
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public abstract class AbstractHlrcXContentTestCase<T extends ToXContent, H> extends AbstractXContentTestCase<T> {
+
+    /**
+     * Generic test that creates new instance of HLRC request/response from the test instance and checks
+     * both for equality and asserts equality on the two queries.
+     */
+    public final void testHlrcFromXContent() throws IOException {
+        AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, this::createTestInstance, supportsUnknownFields(),
+            getShuffleFieldsExceptions(), getRandomFieldsExcludeFilter(), this::createParser,
+            p -> convertHlrcToInternal(doHlrcParseInstance(p)),
+            this::assertEqualInstances, true, getToXContentParams());
+    }
+
+    /**
+     * Parses to a new HLRC instance using the provided {@link XContentParser}
+     */
+    public abstract H doHlrcParseInstance(XContentParser parser) throws IOException;
+
+    /**
+     * Converts a HLRC instance to a XPack instance
+     */
+    public abstract T convertHlrcToInternal(H instance);
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java
index fac99959c536a..1e77d6a83f2ed 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java
@@ -7,12 +7,12 @@
 
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet;
 import org.elasticsearch.protocol.xpack.license.LicenseStatus;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -21,8 +21,11 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.io.IOException;
+import java.util.stream.Collectors;
+
+public class XPackInfoResponseTests extends
+        AbstractHlrcStreamableXContentTestCase<XPackInfoResponse, org.elasticsearch.client.xpack.XPackInfoResponse> {
 
-public class XPackInfoResponseTests extends AbstractStreamableXContentTestCase<XPackInfoResponse> {
     @Override
     protected XPackInfoResponse doParseInstance(XContentParser parser) throws IOException {
         return XPackInfoResponse.fromXContent(parser);
@@ -33,6 +36,38 @@ protected XPackInfoResponse createBlankInstance() {
         return new XPackInfoResponse();
     }
 
+    @Override
+    public org.elasticsearch.client.xpack.XPackInfoResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.xpack.XPackInfoResponse.fromXContent(parser);
+    }
+
+    @Override
+    public XPackInfoResponse convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse instance) {
+        return new XPackInfoResponse(convertHlrcToInternal(instance.getBuildInfo()),
+            convertHlrcToInternal(instance.getLicenseInfo()), convertHlrcToInternal(instance.getFeatureSetsInfo()));
+    }
+
+    private BuildInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse.BuildInfo buildInfo) {
+        return buildInfo != null ? new BuildInfo(buildInfo.getHash(), buildInfo.getTimestamp()) : null;
+    }
+
+    private LicenseInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse.LicenseInfo licenseInfo) {
+        return licenseInfo != null
+            ? new LicenseInfo(licenseInfo.getUid(), licenseInfo.getType(), licenseInfo.getMode(),
+                licenseInfo.getStatus() != null ? LicenseStatus.valueOf(licenseInfo.getStatus().name()) : null,
+                licenseInfo.getExpiryDate())
+            : null;
+    }
+
+    private FeatureSetsInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse.FeatureSetsInfo featureSetsInfo) {
+        return featureSetsInfo != null
+            ? new FeatureSetsInfo(featureSetsInfo.getFeatureSets().values().stream()
+            .map(fs -> new FeatureSet(fs.name(), fs.description(), fs.available(), fs.enabled(),
+                fs.nativeCodeInfo()))
+            .collect(Collectors.toSet()))
+            : null;
+    }
+
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
         return path -> path.equals("features")
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java
index 34d96440b204d..4acac1bd34f36 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/graph/GraphExploreResponseTests.java
@@ -10,24 +10,74 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.protocol.AbstractHlrcXContentTestCase;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class GraphExploreResponseTests extends AbstractXContentTestCase< GraphExploreResponse> {
+public class GraphExploreResponseTests extends
+        AbstractHlrcXContentTestCase<GraphExploreResponse, org.elasticsearch.client.graph.GraphExploreResponse> {
+
+    static final Function<org.elasticsearch.client.graph.Vertex.VertexId, Vertex.VertexId> VERTEX_ID_FUNCTION =
+        vId -> new Vertex.VertexId(vId.getField(), vId.getTerm());
+    static final Function<org.elasticsearch.client.graph.Vertex, Vertex> VERTEX_FUNCTION =
+        v -> new Vertex(v.getField(), v.getTerm(), v.getWeight(), v.getHopDepth(), v.getBg(), v.getFg());
+
+    @Override
+    public org.elasticsearch.client.graph.GraphExploreResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.graph.GraphExploreResponse.fromXContent(parser);
+    }
+
+    @Override
+    public GraphExploreResponse convertHlrcToInternal(org.elasticsearch.client.graph.GraphExploreResponse instance) {
+        return new GraphExploreResponse(instance.getTookInMillis(), instance.isTimedOut(),
+            instance.getShardFailures(), convertVertices(instance), convertConnections(instance), instance.isReturnDetailedInfo());
+    }
+
+    public Map<Vertex.VertexId, Vertex> convertVertices(org.elasticsearch.client.graph.GraphExploreResponse instance) {
+        final Collection<org.elasticsearch.client.graph.Vertex.VertexId> vertexIds = instance.getVertexIds();
+        final Map<Vertex.VertexId, Vertex> vertexMap = new LinkedHashMap<>(vertexIds.size());
+
+        for (org.elasticsearch.client.graph.Vertex.VertexId vertexId : vertexIds) {
+            final org.elasticsearch.client.graph.Vertex vertex = instance.getVertex(vertexId);
+
+            vertexMap.put(VERTEX_ID_FUNCTION.apply(vertexId), VERTEX_FUNCTION.apply(vertex));
+        }
+        return vertexMap;
+    }
+
+    public Map<Connection.ConnectionId, Connection> convertConnections(org.elasticsearch.client.graph.GraphExploreResponse instance) {
+        final Collection<org.elasticsearch.client.graph.Connection.ConnectionId> connectionIds = instance.getConnectionIds();
+        final Map<Connection.ConnectionId, Connection> connectionMap = new LinkedHashMap<>(connectionIds.size());
+        for (org.elasticsearch.client.graph.Connection.ConnectionId connectionId : connectionIds) {
+            final org.elasticsearch.client.graph.Connection connection = instance.getConnection(connectionId);
+            final Connection.ConnectionId connectionId1 =
+                new Connection.ConnectionId(VERTEX_ID_FUNCTION.apply(connectionId.getSource()),
+                    VERTEX_ID_FUNCTION.apply(connectionId.getTarget()));
+            final Connection connection1 = new Connection(VERTEX_FUNCTION.apply(connection.getFrom()),
+                VERTEX_FUNCTION.apply(connection.getTo()),
+                connection.getWeight(), connection.getDocCount());
+            connectionMap.put(connectionId1, connection1);
+        }
+        return connectionMap;
+    }
 
     @Override
     protected  GraphExploreResponse createTestInstance() {
         return createInstance(0);
     }
+
     private static  GraphExploreResponse createInstance(int numFailures) {
         int numItems = randomIntBetween(4, 128);
         boolean timedOut = randomBoolean();
@@ -66,14 +116,14 @@ private static  GraphExploreResponse createInstance(int numFailures) {
     protected String[] getShuffleFieldsExceptions() {
         return new String[]{"vertices", "connections"};
     }
-    
+
     private static   GraphExploreResponse createTestInstanceWithFailures() {
         return createInstance(randomIntBetween(1, 128));
     }
 
     @Override
     protected  GraphExploreResponse doParseInstance(XContentParser parser) throws IOException {
-        return GraphExploreResponse.fromXContext(parser);
+        return GraphExploreResponse.fromXContent(parser);
     }
 
     @Override
@@ -84,7 +134,7 @@ protected boolean supportsUnknownFields() {
     @Override
     protected boolean assertToXContentEquivalence() {
         return false;
-    }     
+    }
 
     protected Predicate<String> getRandomFieldsExcludeFilterWhenResultHasErrors() {
         return field -> field.startsWith("responses");
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java
index 7149477d00765..0b73850c5e637 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java
@@ -9,9 +9,25 @@
 
 import org.elasticsearch.test.ESTestCase;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+
 public class LicenseStatusTests extends ESTestCase {
     public void testSerialization() throws IOException {
         LicenseStatus status = randomFrom(LicenseStatus.values());
         assertSame(status, copyWriteable(status, writableRegistry(), LicenseStatus::readFrom));
     }
+
+    public void testCompatibility() {
+        final LicenseStatus[] values = LicenseStatus.values();
+        final org.elasticsearch.client.license.LicenseStatus[] hlrcValues =
+            org.elasticsearch.client.license.LicenseStatus.values();
+
+        assertThat(values.length, equalTo(hlrcValues.length));
+
+        for (LicenseStatus value : values) {
+            final org.elasticsearch.client.license.LicenseStatus licenseStatus =
+                org.elasticsearch.client.license.LicenseStatus.fromString(value.label());
+            assertThat(licenseStatus.label(), equalTo(value.label()));
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java
index a09fd6fb99b45..87ba4324ec1e3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponseTests.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.protocol.xpack.license;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -15,7 +15,19 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 
-public class PutLicenseResponseTests extends AbstractStreamableXContentTestCase<PutLicenseResponse> {
+public class PutLicenseResponseTests extends
+        AbstractHlrcStreamableXContentTestCase<PutLicenseResponse, org.elasticsearch.client.license.PutLicenseResponse> {
+
+    @Override
+    public org.elasticsearch.client.license.PutLicenseResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.license.PutLicenseResponse.fromXContent(parser);
+    }
+
+    @Override
+    public PutLicenseResponse convertHlrcToInternal(org.elasticsearch.client.license.PutLicenseResponse instance) {
+        return new PutLicenseResponse(instance.isAcknowledged(), LicensesStatus.valueOf(instance.status().name()),
+            instance.acknowledgeHeader(), instance.acknowledgeMessages());
+    }
 
     @Override
     protected boolean supportsUnknownFields() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/StartBasicResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/StartBasicResponseTests.java
new file mode 100644
index 0000000000000..78e1c75483aa6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/license/StartBasicResponseTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.protocol.xpack.license;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.PostStartBasicResponse;
+import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
+
+public class StartBasicResponseTests extends
+        AbstractHlrcStreamableXContentTestCase<PostStartBasicResponse, org.elasticsearch.client.license.StartBasicResponse> {
+
+    @Override
+    public org.elasticsearch.client.license.StartBasicResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.license.StartBasicResponse.fromXContent(parser);
+    }
+
+    @Override
+    public PostStartBasicResponse convertHlrcToInternal(org.elasticsearch.client.license.StartBasicResponse instance) {
+        return new PostStartBasicResponse(PostStartBasicResponse.Status.valueOf(instance.getStatus().name()),
+            instance.getAcknowledgeMessages(), instance.getAcknowledgeMessage());
+    }
+
+    @Override
+    protected PostStartBasicResponse doParseInstance(XContentParser parser) throws IOException {
+        return PostStartBasicResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected PostStartBasicResponse createBlankInstance() {
+        return new PostStartBasicResponse();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // The structure of the response is such that unknown fields inside acknowledge cannot be supported since they
+        // are treated as messages from new services
+        return p -> p.startsWith("acknowledge");
+    }
+
+    @Override
+    protected PostStartBasicResponse createTestInstance() {
+        PostStartBasicResponse.Status status = randomFrom(PostStartBasicResponse.Status.values());
+        String acknowledgeMessage = null;
+        Map<String, String[]> ackMessages = Collections.emptyMap();
+        if (status != PostStartBasicResponse.Status.GENERATED_BASIC) {
+            acknowledgeMessage = randomAlphaOfLength(10);
+            ackMessages = randomAckMessages();
+        }
+        final PostStartBasicResponse postStartBasicResponse = new PostStartBasicResponse(status, ackMessages, acknowledgeMessage);
+        logger.info("{}", Strings.toString(postStartBasicResponse));
+        return postStartBasicResponse;
+    }
+
+    private static Map<String, String[]> randomAckMessages() {
+        int nFeatures = randomIntBetween(1, 5);
+
+        Map<String, String[]> ackMessages = new HashMap<>();
+
+        for (int i = 0; i < nFeatures; i++) {
+            String feature = randomAlphaOfLengthBetween(9, 15);
+            int nMessages = randomIntBetween(1, 5);
+            String[] messages = new String[nMessages];
+            for (int j = 0; j < nMessages; j++) {
+                messages[j] = randomAlphaOfLengthBetween(10, 30);
+            }
+            ackMessages.put(feature, messages);
+        }
+
+        return ackMessages;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java
index 57f01a4454e02..76f00ebb24309 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/migration/IndexUpgradeInfoResponseTests.java
@@ -6,18 +6,36 @@
 package org.elasticsearch.protocol.xpack.migration;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
 
+import java.io.IOException;
+import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.stream.Collectors;
+
+public class IndexUpgradeInfoResponseTests extends
+        AbstractHlrcStreamableXContentTestCase<IndexUpgradeInfoResponse, org.elasticsearch.client.migration.IndexUpgradeInfoResponse> {
 
-public class IndexUpgradeInfoResponseTests extends AbstractStreamableXContentTestCase<IndexUpgradeInfoResponse> {
     @Override
     protected IndexUpgradeInfoResponse doParseInstance(XContentParser parser) {
         return IndexUpgradeInfoResponse.fromXContent(parser);
     }
 
+    @Override
+    public org.elasticsearch.client.migration.IndexUpgradeInfoResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.migration.IndexUpgradeInfoResponse.fromXContent(parser);
+    }
+
+    @Override
+    public IndexUpgradeInfoResponse convertHlrcToInternal(org.elasticsearch.client.migration.IndexUpgradeInfoResponse instance) {
+        final Map<String, org.elasticsearch.client.migration.UpgradeActionRequired> actions = instance.getActions();
+        return new IndexUpgradeInfoResponse(actions.entrySet().stream().map(
+            e -> new AbstractMap.SimpleEntry<>(e.getKey(), UpgradeActionRequired.valueOf(e.getValue().name()))
+        ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+    }
+
     @Override
     protected IndexUpgradeInfoResponse createBlankInstance() {
         return new IndexUpgradeInfoResponse();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
index 209bc790a8c54..7486252f53859 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/DeleteWatchResponseTests.java
@@ -6,11 +6,12 @@
 package org.elasticsearch.protocol.xpack.watcher;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.protocol.AbstractHlrcXContentTestCase;
 
 import java.io.IOException;
 
-public class DeleteWatchResponseTests extends AbstractXContentTestCase<DeleteWatchResponse> {
+public class DeleteWatchResponseTests extends
+        AbstractHlrcXContentTestCase<DeleteWatchResponse, org.elasticsearch.client.watcher.DeleteWatchResponse> {
 
     @Override
     protected DeleteWatchResponse createTestInstance() {
@@ -25,6 +26,16 @@ protected DeleteWatchResponse doParseInstance(XContentParser parser) throws IOEx
         return DeleteWatchResponse.fromXContent(parser);
     }
 
+    @Override
+    public org.elasticsearch.client.watcher.DeleteWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.watcher.DeleteWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    public DeleteWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.DeleteWatchResponse instance) {
+        return new DeleteWatchResponse(instance.getId(), instance.getVersion(), instance.isFound());
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java
index 1fc2f61b684c7..8ea4a84daed95 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/PutWatchResponseTests.java
@@ -6,11 +6,12 @@
 package org.elasticsearch.protocol.xpack.watcher;
 
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.protocol.AbstractHlrcXContentTestCase;
 
 import java.io.IOException;
 
-public class PutWatchResponseTests extends AbstractXContentTestCase<PutWatchResponse> {
+public class PutWatchResponseTests extends
+        AbstractHlrcXContentTestCase<PutWatchResponse, org.elasticsearch.client.watcher.PutWatchResponse> {
 
     @Override
     protected PutWatchResponse createTestInstance() {
@@ -25,6 +26,16 @@ protected PutWatchResponse doParseInstance(XContentParser parser) throws IOExcep
         return PutWatchResponse.fromXContent(parser);
     }
 
+    @Override
+    public org.elasticsearch.client.watcher.PutWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.watcher.PutWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    public PutWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.PutWatchResponse instance) {
+        return new PutWatchResponse(instance.getId(), instance.getVersion(), instance.isCreated());
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractActionTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractActionTestCase.java
new file mode 100644
index 0000000000000..bed04a7cf5425
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractActionTestCase.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public abstract class AbstractActionTestCase<T extends LifecycleAction> extends AbstractSerializingTestCase<T> {
+
+    public abstract void testToSteps();
+
+    protected boolean isSafeAction() {
+        return true;
+    }
+
+    public final void testIsSafeAction() {
+        LifecycleAction action = createTestInstance();
+        assertEquals(isSafeAction(), action.isSafeAction());
+    }
+
+    public void testToStepKeys() {
+        T action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        List<StepKey> stepKeys = action.toStepKeys(phase);
+        assertNotNull(stepKeys);
+        List<StepKey> expectedStepKeys = steps.stream().map(Step::getKey).collect(Collectors.toList());
+        assertEquals(expectedStepKeys, stepKeys);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractStepTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractStepTestCase.java
new file mode 100644
index 0000000000000..2757a0499aae9
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AbstractStepTestCase.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public abstract class AbstractStepTestCase<T extends Step> extends ESTestCase {
+
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    protected abstract T createRandomInstance();
+    protected abstract T mutateInstance(T instance);
+    protected abstract T copyInstance(T instance);
+
+    public void testHashcodeAndEquals() {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createRandomInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+
+    public static StepKey randomStepKey() {
+        String randomPhase = randomAlphaOfLength(10);
+        String randomAction = randomAlphaOfLength(10);
+        String randomStepName = randomAlphaOfLength(10);
+        return new StepKey(randomPhase, randomAction, randomStepName);
+    }
+
+    public void testStepNameNotError() {
+        T instance = createRandomInstance();
+        StepKey stepKey = instance.getKey();
+        assertFalse(ErrorStep.NAME.equals(stepKey.getName()));
+        StepKey nextStepKey = instance.getKey();
+        assertFalse(ErrorStep.NAME.equals(nextStepKey.getName()));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateActionTests.java
new file mode 100644
index 0000000000000..dfe2afc5d19ef
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocateActionTests.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AllocateActionTests extends AbstractActionTestCase<AllocateAction> {
+
+    @Override
+    protected AllocateAction doParseInstance(XContentParser parser) {
+        return AllocateAction.parse(parser);
+    }
+
+    @Override
+    protected AllocateAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static AllocateAction randomInstance() {
+        boolean hasAtLeastOneMap = false;
+        Map<String, String> includes;
+        if (randomBoolean()) {
+            includes = randomMap(1, 100);
+            hasAtLeastOneMap = true;
+        } else {
+            includes = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Map<String, String> excludes;
+        if (randomBoolean()) {
+            hasAtLeastOneMap = true;
+            excludes = randomMap(1, 100);
+        } else {
+            excludes = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Map<String, String> requires;
+        if (hasAtLeastOneMap == false || randomBoolean()) {
+            requires = randomMap(1, 100);
+        } else {
+            requires = randomBoolean() ? null : Collections.emptyMap();
+        }
+        Integer numberOfReplicas = randomBoolean() ? null : randomIntBetween(0, 10);
+        return new AllocateAction(numberOfReplicas, includes, excludes, requires);
+    }
+
+
+    @Override
+    protected Reader<AllocateAction> instanceReader() {
+        return AllocateAction::new;
+    }
+
+    @Override
+    protected AllocateAction mutateInstance(AllocateAction instance) {
+        Map<String, String> include = instance.getInclude();
+        Map<String, String> exclude = instance.getExclude();
+        Map<String, String> require = instance.getRequire();
+        Integer numberOfReplicas = instance.getNumberOfReplicas();
+        switch (randomIntBetween(0, 3)) {
+        case 0:
+            include = new HashMap<>(include);
+            include.put(randomAlphaOfLengthBetween(11, 15), randomAlphaOfLengthBetween(1, 20));
+            break;
+        case 1:
+            exclude = new HashMap<>(exclude);
+            exclude.put(randomAlphaOfLengthBetween(11, 15), randomAlphaOfLengthBetween(1, 20));
+            break;
+        case 2:
+            require = new HashMap<>(require);
+            require.put(randomAlphaOfLengthBetween(11, 15), randomAlphaOfLengthBetween(1, 20));
+            break;
+        case 3:
+            numberOfReplicas = randomIntBetween(11, 20);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new AllocateAction(numberOfReplicas, include, exclude, require);
+    }
+
+    public void testAllMapsNullOrEmpty() {
+        Map<String, String> include = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> exclude = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> require = randomBoolean() ? null : Collections.emptyMap();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> new AllocateAction(null, include, exclude, require));
+        assertEquals("At least one of " + AllocateAction.INCLUDE_FIELD.getPreferredName() + ", "
+                + AllocateAction.EXCLUDE_FIELD.getPreferredName() + " or " + AllocateAction.REQUIRE_FIELD.getPreferredName()
+                + "must contain attributes for action " + AllocateAction.NAME, exception.getMessage());
+    }
+
+    public void testInvalidNumberOfReplicas() {
+        Map<String, String> include = randomMap(1, 5);
+        Map<String, String> exclude = randomBoolean() ? null : Collections.emptyMap();
+        Map<String, String> require = randomBoolean() ? null : Collections.emptyMap();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> new AllocateAction(randomIntBetween(-1000, -1), include, exclude, require));
+        assertEquals("[" + AllocateAction.NUMBER_OF_REPLICAS_FIELD.getPreferredName() + "] must be >= 0", exception.getMessage());
+    }
+
+    public static Map<String, String> randomMap(int minEntries, int maxEntries) {
+        Map<String, String> map = new HashMap<>();
+        int numIncludes = randomIntBetween(minEntries, maxEntries);
+        for (int i = 0; i < numIncludes; i++) {
+            map.put(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        }
+        return map;
+    }
+
+    public void testToSteps() {
+        AllocateAction action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        assertEquals(2, steps.size());
+        StepKey expectedFirstStepKey = new StepKey(phase, AllocateAction.NAME, AllocateAction.NAME);
+        StepKey expectedSecondStepKey = new StepKey(phase, AllocateAction.NAME, AllocationRoutedStep.NAME);
+        UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
+        assertEquals(expectedFirstStepKey, firstStep.getKey());
+        assertEquals(expectedSecondStepKey, firstStep.getNextStepKey());
+        Settings.Builder expectedSettings = Settings.builder();
+        if (action.getNumberOfReplicas() != null) {
+            expectedSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, action.getNumberOfReplicas());
+        }
+        action.getInclude().forEach(
+            (key, value) -> expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + key, value));
+        action.getExclude().forEach(
+            (key, value) -> expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + key, value));
+        action.getRequire().forEach(
+            (key, value) -> expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + key, value));
+        assertThat(firstStep.getSettings(), equalTo(expectedSettings.build()));
+        AllocationRoutedStep secondStep = (AllocationRoutedStep) steps.get(1);
+        assertEquals(expectedSecondStepKey, secondStep.getKey());
+        assertEquals(nextStepKey, secondStep.getNextStepKey());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepInfoTests.java
new file mode 100644
index 0000000000000..80eb89c45b952
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepInfoTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocationRoutedStep.Info;
+
+import java.io.IOException;
+
+public class AllocationRoutedStepInfoTests extends AbstractXContentTestCase<AllocationRoutedStep.Info> {
+
+    @Override
+    protected Info createTestInstance() {
+        return new Info(randomNonNegativeLong(), randomNonNegativeLong(), randomBoolean());
+    }
+
+    @Override
+    protected Info doParseInstance(XContentParser parser) throws IOException {
+        return Info.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public final void testEqualsAndHashcode() {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+
+    protected final Info copyInstance(Info instance) throws IOException {
+        return new Info(instance.getActualReplicas(), instance.getNumberShardsLeftToAllocate(), instance.allShardsActive());
+    }
+
+    protected Info mutateInstance(Info instance) throws IOException {
+        long actualReplicas = instance.getActualReplicas();
+        long shardsToAllocate = instance.getNumberShardsLeftToAllocate();
+        boolean allShardsActive = instance.allShardsActive();
+        switch (between(0, 2)) {
+        case 0:
+            shardsToAllocate += between(1, 20);
+            break;
+        case 1:
+            allShardsActive = allShardsActive == false;
+            break;
+        case 2:
+            actualReplicas += between(1, 20);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new Info(actualReplicas, shardsToAllocate, allShardsActive);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepTests.java
new file mode 100644
index 0000000000000..03320516d9442
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/AllocationRoutedStepTests.java
@@ -0,0 +1,309 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep.Result;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.Collections;
+import java.util.Map;
+
+public class AllocationRoutedStepTests extends AbstractStepTestCase<AllocationRoutedStep> {
+
+    @Override
+    public AllocationRoutedStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+
+        return new AllocationRoutedStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public AllocationRoutedStep mutateInstance(AllocationRoutedStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new AllocationRoutedStep(key, nextKey);
+    }
+
+    @Override
+    public AllocationRoutedStep copyInstance(AllocationRoutedStep instance) {
+        return new AllocationRoutedStep(instance.getKey(), instance.getNextStepKey());
+    }
+
+    public void testConditionMet() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> includes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> excludes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        includes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        excludes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+        });
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED));
+
+        AllocationRoutedStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(true, null));
+    }
+
+    public void testConditionMetOnlyOneCopyAllocated() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> includes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> excludes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        includes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        excludes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+        });
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                        ShardRoutingState.STARTED));
+
+        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, 1, true)));
+    }
+
+    public void testExecuteAllocateNotComplete() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> includes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> excludes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        includes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        excludes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+        });
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), "node2", true, ShardRoutingState.STARTED));
+
+        AllocationRoutedStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, 1, true)));
+    }
+
+    public void testExecuteAllocateNotCompleteOnlyOneCopyAllocated() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> includes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> excludes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        includes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        excludes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+        });
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                        ShardRoutingState.STARTED));
+
+        AllocationRoutedStep step = new AllocationRoutedStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, 1, true)));
+    }
+
+    public void testExecuteAllocateUnassigned() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> includes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> excludes = AllocateActionTests.randomMap(1, 5);
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+                .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        includes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        excludes.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + k, v);
+        });
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), null, null, true, ShardRoutingState.UNASSIGNED,
+                        new UnassignedInfo(randomFrom(Reason.values()), "the shard is intentionally unassigned")));
+
+        AllocationRoutedStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(0, -1, false)));
+    }
+
+    /**
+     * this  tests the scenario where
+     *
+     * PUT index
+     * {
+     *  "settings": {
+     *   "number_of_replicas": 0,
+     *   "number_of_shards": 1
+     *  }
+     * }
+     *
+     * PUT index/_settings
+     * {
+     *  "number_of_replicas": 1,
+     *  "index.routing.allocation.include._name": "{node-name}"
+     * }
+     */
+    public void testExecuteReplicasNotAllocatedOnSingleNode() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = Collections.singletonMap("_name", "node1");
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), null, null, false, ShardRoutingState.UNASSIGNED,
+                new UnassignedInfo(Reason.REPLICA_ADDED, "no attempt")));
+
+        AllocationRoutedStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 1, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new AllocationRoutedStep.Info(1, -1, false)));
+    }
+
+    public void testExecuteIndexMissing() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).build();
+
+        AllocationRoutedStep step = createRandomInstance();
+
+        Result actualResult = step.isConditionMet(index, clusterState);
+        assertFalse(actualResult.isComplete());
+        assertNull(actualResult.getInfomationContext());
+    }
+
+    private void assertAllocateStatus(Index index, int shards, int replicas, AllocationRoutedStep step, Settings.Builder existingSettings,
+            Settings.Builder node1Settings, Settings.Builder node2Settings, IndexRoutingTable.Builder indexRoutingTable,
+            ClusterStateWaitStep.Result expectedResult) {
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName()).settings(existingSettings).numberOfShards(shards)
+                .numberOfReplicas(replicas).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+                indexMetadata);
+
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+                .nodes(DiscoveryNodes.builder()
+                        .add(DiscoveryNode.createLocal(node1Settings.build(), new TransportAddress(TransportAddress.META_ADDRESS, 9200),
+                                "node1"))
+                        .add(DiscoveryNode.createLocal(node2Settings.build(), new TransportAddress(TransportAddress.META_ADDRESS, 9201),
+                                "node2")))
+                .routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+        Result actualResult = step.isConditionMet(index, clusterState);
+        assertEquals(expectedResult.isComplete(), actualResult.isComplete());
+        assertEquals(expectedResult.getInfomationContext(), actualResult.getInfomationContext());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStepTests.java
new file mode 100644
index 0000000000000..f7a6687dd987c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CheckShrinkReadyStepTests.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class CheckShrinkReadyStepTests extends AbstractStepTestCase<CheckShrinkReadyStep> {
+
+    @Override
+    public CheckShrinkReadyStep createRandomInstance() {
+        Step.StepKey stepKey = randomStepKey();
+        Step.StepKey nextStepKey = randomStepKey();
+
+        return new CheckShrinkReadyStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public CheckShrinkReadyStep mutateInstance(CheckShrinkReadyStep instance) {
+        Step.StepKey key = instance.getKey();
+        Step.StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+            case 0:
+                key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new CheckShrinkReadyStep(key, nextKey);
+    }
+
+    @Override
+    public CheckShrinkReadyStep copyInstance(CheckShrinkReadyStep instance) {
+        return new CheckShrinkReadyStep(instance.getKey(), instance.getNextStepKey());
+    }
+
+    public void testNoSetting() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+                new ClusterStateWaitStep.Result(true, null));
+        });
+        assertThat(e.getMessage(), containsString("Cannot check shrink allocation as there are no allocation rules by _id"));
+    }
+
+    public void testConditionMet() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(true, null));
+    }
+
+    public void testConditionMetOnlyOneCopyAllocated() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(true, null));
+    }
+
+    public void testConditionNotMetDueToRelocation() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+        boolean primaryOnNode1 = randomBoolean();
+        ShardRouting shardOnNode1 = TestShardRouting.newShardRouting(new ShardId(index, 0),
+            "node1", primaryOnNode1, ShardRoutingState.STARTED);
+        shardOnNode1 = shardOnNode1.relocate("node3", 230);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(shardOnNode1)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 1, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 1, 1)));
+    }
+
+    public void testExecuteAllocateNotComplete() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), "node2", true, ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 2, 1)));
+    }
+
+    public void testExecuteAllocateNotCompleteOnlyOneCopyAllocated() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        boolean primaryOnNode1 = randomBoolean();
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", primaryOnNode1, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node2", primaryOnNode1 == false,
+                ShardRoutingState.STARTED));
+
+        CheckShrinkReadyStep step = new CheckShrinkReadyStep(randomStepKey(), randomStepKey());
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("node1", 2, 1)));
+    }
+
+    public void testExecuteAllocateUnassigned() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = AllocateActionTests.randomMap(1, 5);
+        Settings.Builder existingSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._id", "node1")
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            existingSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+            node1Settings.put(Node.NODE_ATTRIBUTES.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), null, null, true, ShardRoutingState.UNASSIGNED,
+                new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "the shard is intentionally unassigned")));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 2, 0, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("", 2, -1)));
+    }
+
+    /**
+     * this  tests the scenario where
+     *
+     * PUT index
+     * {
+     *  "settings": {
+     *   "number_of_replicas": 0,
+     *   "number_of_shards": 1
+     *  }
+     * }
+     *
+     * PUT index/_settings
+     * {
+     *  "number_of_replicas": 1,
+     *  "index.routing.allocation.include._id": "{node-name}"
+     * }
+     */
+    public void testExecuteReplicasNotAllocatedOnSingleNode() {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Map<String, String> requires = Collections.singletonMap("_id", "node1");
+        Settings.Builder existingSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT.id)
+            .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID());
+        Settings.Builder expectedSettings = Settings.builder();
+        Settings.Builder node1Settings = Settings.builder();
+        Settings.Builder node2Settings = Settings.builder();
+        requires.forEach((k, v) -> {
+            expectedSettings.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + k, v);
+        });
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), null, null, false, ShardRoutingState.UNASSIGNED,
+                new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, "no attempt")));
+
+        CheckShrinkReadyStep step = createRandomInstance();
+        assertAllocateStatus(index, 1, 1, step, existingSettings, node1Settings, node2Settings, indexRoutingTable,
+            new ClusterStateWaitStep.Result(false, new CheckShrinkReadyStep.Info("", 1, -1)));
+    }
+
+    public void testExecuteIndexMissing() throws Exception {
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).build();
+
+        CheckShrinkReadyStep step = createRandomInstance();
+
+        ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState);
+        assertFalse(actualResult.isComplete());
+        assertNull(actualResult.getInfomationContext());
+    }
+
+    private void assertAllocateStatus(Index index, int shards, int replicas, CheckShrinkReadyStep step, Settings.Builder existingSettings,
+                                      Settings.Builder node1Settings, Settings.Builder node2Settings,
+                                      IndexRoutingTable.Builder indexRoutingTable, ClusterStateWaitStep.Result expectedResult) {
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName()).settings(existingSettings).numberOfShards(shards)
+            .numberOfReplicas(replicas).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+            indexMetadata);
+
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+            .nodes(DiscoveryNodes.builder()
+                .add(DiscoveryNode.createLocal(Settings.builder().put(node1Settings.build())
+                        .put(Node.NODE_NAME_SETTING.getKey(), "node1").build(),
+                    new TransportAddress(TransportAddress.META_ADDRESS, 9200),
+                    "node1"))
+                .add(DiscoveryNode.createLocal(Settings.builder().put(node2Settings.build())
+                        .put(Node.NODE_NAME_SETTING.getKey(), "node2").build(),
+                    new TransportAddress(TransportAddress.META_ADDRESS, 9201),
+                    "node2")))
+            .routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+        ClusterStateWaitStep.Result actualResult = step.isConditionMet(index, clusterState);
+        assertEquals(expectedResult.isComplete(), actualResult.isComplete());
+        assertEquals(expectedResult.getInfomationContext(), actualResult.getInfomationContext());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStepTests.java
new file mode 100644
index 0000000000000..40dd022c05de6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStepTests.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.Map;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionStateTests.createCustomMetadata;
+
+public class CopyExecutionStateStepTests extends AbstractStepTestCase<CopyExecutionStateStep> {
+    @Override
+    protected CopyExecutionStateStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new CopyExecutionStateStep(stepKey, nextStepKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    protected CopyExecutionStateStep mutateInstance(CopyExecutionStateStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+
+        switch (between(0, 2)) {
+            case 0:
+                key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 2:
+                shrunkIndexPrefix += randomAlphaOfLength(5);
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new CopyExecutionStateStep(key, nextKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    protected CopyExecutionStateStep copyInstance(CopyExecutionStateStep instance) {
+        return new CopyExecutionStateStep(instance.getKey(), instance.getNextStepKey(), instance.getShrunkIndexPrefix());
+    }
+
+    public void testPerformAction() {
+        CopyExecutionStateStep step = createRandomInstance();
+        String indexName = randomAlphaOfLengthBetween(5, 20);
+        Map<String, String> customMetadata = createCustomMetadata();
+
+        IndexMetaData originalIndexMetaData = IndexMetaData.builder(indexName)
+            .settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1,5))
+            .numberOfReplicas(randomIntBetween(1,5))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, customMetadata)
+            .build();
+        IndexMetaData shrunkIndexMetaData = IndexMetaData.builder(step.getShrunkIndexPrefix() + indexName)
+            .settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1,5))
+            .numberOfReplicas(randomIntBetween(1,5))
+            .build();
+        ClusterState originalClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder()
+                .put(originalIndexMetaData, false)
+                .put(shrunkIndexMetaData, false))
+            .build();
+
+        ClusterState newClusterState = step.performAction(originalIndexMetaData.getIndex(), originalClusterState);
+
+        LifecycleExecutionState oldIndexData = LifecycleExecutionState.fromIndexMetadata(originalIndexMetaData);
+        LifecycleExecutionState newIndexData = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(step.getShrunkIndexPrefix() + indexName));
+
+        assertEquals(oldIndexData.getLifecycleDate(), newIndexData.getLifecycleDate());
+        assertEquals(oldIndexData.getPhase(), newIndexData.getPhase());
+        assertEquals(oldIndexData.getAction(), newIndexData.getAction());
+        assertEquals(ShrunkenIndexCheckStep.NAME, newIndexData.getStep());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteActionTests.java
new file mode 100644
index 0000000000000..3286ce0225a39
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteActionTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.List;
+
+public class DeleteActionTests extends AbstractActionTestCase<DeleteAction> {
+
+    @Override
+    protected DeleteAction doParseInstance(XContentParser parser) throws IOException {
+        return DeleteAction.parse(parser);
+    }
+
+    @Override
+    protected DeleteAction createTestInstance() {
+        return new DeleteAction();
+    }
+
+    @Override
+    protected Reader<DeleteAction> instanceReader() {
+        return DeleteAction::new;
+    }
+
+    public void testToSteps() {
+        DeleteAction action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        assertEquals(1, steps.size());
+        StepKey expectedFirstStepKey = new StepKey(phase, DeleteAction.NAME, DeleteStep.NAME);
+        DeleteStep firstStep = (DeleteStep) steps.get(0);
+        assertEquals(expectedFirstStepKey, firstStep.getKey());
+        assertEquals(nextStepKey, firstStep.getNextStepKey());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStepTests.java
new file mode 100644
index 0000000000000..c85df6de659e8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteStepTests.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeleteStepTests extends AbstractStepTestCase<DeleteStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    public DeleteStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+
+        return new DeleteStep(stepKey, nextStepKey, client);
+    }
+
+    @Override
+    public DeleteStep mutateInstance(DeleteStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new DeleteStep(key, nextKey, instance.getClient());
+    }
+
+    @Override
+    public DeleteStep copyInstance(DeleteStep instance) {
+        return new DeleteStep(instance.getKey(), instance.getNextStepKey(), instance.getClient());
+    }
+
+    public void testIndexSurvives() {
+        assertFalse(createRandomInstance().indexSurvives());
+    }
+
+    public void testDeleted() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(invocation -> {
+                DeleteIndexRequest request = (DeleteIndexRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertNotNull(request);
+                assertEquals(1, request.indices().length);
+                assertEquals(indexMetaData.getIndex().getName(), request.indices()[0]);
+                listener.onResponse(null);
+                return null;
+        }).when(indicesClient).delete(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+
+        DeleteStep step = createRandomInstance();
+        step.performAction(indexMetaData, null, new AsyncActionStep.Listener() {
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+
+        assertThat(actionCompleted.get(), equalTo(true));
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).delete(Mockito.any(), Mockito.any());
+    }
+
+    public void testExceptionThrown() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                DeleteIndexRequest request = (DeleteIndexRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertNotNull(request);
+                assertEquals(1, request.indices().length);
+                assertEquals(indexMetaData.getIndex().getName(), request.indices()[0]);
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).delete(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        DeleteStep step = createRandomInstance();
+        step.performAction(indexMetaData, null, new AsyncActionStep.Listener() {
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertThat(exceptionThrown.get(), equalTo(true));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStepTests.java
new file mode 100644
index 0000000000000..5bd0cdf230da5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ErrorStepTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public class ErrorStepTests extends AbstractStepTestCase<ErrorStep> {
+
+    @Override
+    public ErrorStep createRandomInstance() {
+        StepKey stepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), ErrorStep.NAME);
+        return new ErrorStep(stepKey);
+    }
+
+    @Override
+    public ErrorStep mutateInstance(ErrorStep instance) {
+        StepKey key = instance.getKey();
+        assertSame(instance.getNextStepKey(), instance.getKey());
+
+        key = new StepKey(key.getPhase(), key.getAction() + randomAlphaOfLength(5), key.getName());
+
+        return new ErrorStep(key);
+    }
+
+    @Override
+    public ErrorStep copyInstance(ErrorStep instance) {
+        assertSame(instance.getNextStepKey(), instance.getKey());
+        return new ErrorStep(instance.getKey());
+    }
+
+    public void testInvalidStepKey() {
+        StepKey invalidKey = randomStepKey();
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new ErrorStep(invalidKey));
+        assertEquals("An error step must have a step key whose step name is " + ErrorStep.NAME, exception.getMessage());
+    }
+
+    @Override
+    public void testStepNameNotError() {
+        // Need to override this test because this is the one special step that
+        // is allowed to have ERROR as the step name
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java
new file mode 100644
index 0000000000000..e520e0d1f19fb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ExplainLifecycleRequestTests extends AbstractWireSerializingTestCase<ExplainLifecycleRequest> {
+
+    @Override
+    protected ExplainLifecycleRequest createTestInstance() {
+        ExplainLifecycleRequest request = new ExplainLifecycleRequest();
+        if (randomBoolean()) {
+            request.indices(generateRandomStringArray(20, 20, false, true));
+        }
+        if (randomBoolean()) {
+            IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            request.indicesOptions(indicesOptions);
+        }
+        return request;
+    }
+
+    @Override
+    protected ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance) throws IOException {
+        String[] indices = instance.indices();
+        IndicesOptions indicesOptions = instance.indicesOptions();
+        switch (between(0, 1)) {
+        case 0:
+            indices = randomValueOtherThanMany(i -> Arrays.equals(i, instance.indices()),
+                    () -> generateRandomStringArray(20, 10, false, true));
+            break;
+        case 1:
+            indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest();
+        newRequest.indices(indices);
+        newRequest.indicesOptions(indicesOptions);
+        return newRequest;
+    }
+
+    @Override
+    protected Reader<ExplainLifecycleRequest> instanceReader() {
+        return ExplainLifecycleRequest::new;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponseTests.java
new file mode 100644
index 0000000000000..8b64e1128c020
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleResponseTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ExplainLifecycleResponseTests extends AbstractStreamableXContentTestCase<ExplainLifecycleResponse> {
+
+    @Override
+    protected ExplainLifecycleResponse createTestInstance() {
+        Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>();
+        for (int i = 0; i < randomIntBetween(0, 2); i++) {
+            IndexLifecycleExplainResponse indexResponse = IndexExplainResponseTests.randomIndexExplainResponse();
+            indexResponses.put(indexResponse.getIndex(), indexResponse);
+        }
+        return new ExplainLifecycleResponse(indexResponses);
+    }
+
+    @Override
+    protected ExplainLifecycleResponse createBlankInstance() {
+        return new ExplainLifecycleResponse();
+    }
+
+    @Override
+    protected ExplainLifecycleResponse mutateInstance(ExplainLifecycleResponse response) {
+        Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>(response.getIndexResponses());
+        IndexLifecycleExplainResponse indexResponse = IndexExplainResponseTests.randomIndexExplainResponse();
+        indexResponses.put(indexResponse.getIndex(), indexResponse);
+        return new ExplainLifecycleResponse(indexResponses);
+    }
+
+    @Override
+    protected ExplainLifecycleResponse doParseInstance(XContentParser parser) throws IOException {
+        return ExplainLifecycleResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Arrays
+            .asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new)));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME), MockAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeActionTests.java
new file mode 100644
index 0000000000000..3772124385c46
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeActionTests.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeAction> {
+
+    @Override
+    protected ForceMergeAction doParseInstance(XContentParser parser) {
+        return ForceMergeAction.parse(parser);
+    }
+
+    @Override
+    protected ForceMergeAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static ForceMergeAction randomInstance() {
+        return new ForceMergeAction(randomIntBetween(1, 100));
+    }
+
+    @Override
+    protected ForceMergeAction mutateInstance(ForceMergeAction instance) {
+        int maxNumSegments = instance.getMaxNumSegments();
+        maxNumSegments = maxNumSegments + randomIntBetween(1, 10);
+        return new ForceMergeAction(maxNumSegments);
+    }
+
+    @Override
+    protected Reader<ForceMergeAction> instanceReader() {
+        return ForceMergeAction::new;
+    }
+
+    public void testMissingMaxNumSegments() throws IOException {
+        BytesReference emptyObject = BytesReference.bytes(JsonXContent.contentBuilder().startObject().endObject());
+        XContentParser parser = XContentHelper.createParser(null, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            emptyObject, XContentType.JSON);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ForceMergeAction.parse(parser));
+        assertThat(e.getMessage(), equalTo("Required [max_num_segments]"));
+    }
+
+    public void testInvalidNegativeSegmentNumber() {
+        Exception r = expectThrows(IllegalArgumentException.class, () -> new ForceMergeAction(randomIntBetween(-10, 0)));
+        assertThat(r.getMessage(), equalTo("[max_num_segments] must be a positive integer"));
+    }
+
+    public void testToSteps() {
+        ForceMergeAction instance = createTestInstance();
+        String phase = randomAlphaOfLength(5);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10));
+        List<Step> steps = instance.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        assertEquals(3, steps.size());
+        UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
+        ForceMergeStep secondStep = (ForceMergeStep) steps.get(1);
+        SegmentCountStep thirdStep = (SegmentCountStep) steps.get(2);
+        assertThat(firstStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME)));
+        assertThat(firstStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME)));
+        assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(firstStep.getSettings()));
+        assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME)));
+        assertThat(secondStep.getNextStepKey(), equalTo(thirdStep.getKey()));
+        assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME)));
+        assertThat(thirdStep.getNextStepKey(), equalTo(nextStepKey));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStepTests.java
new file mode 100644
index 0000000000000..9a38ddf3a2677
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeStepTests.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.mockito.Mockito;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ForceMergeStepTests extends AbstractStepTestCase<ForceMergeStep> {
+
+    @Override
+    public ForceMergeStep createRandomInstance() {
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int maxNumSegments = randomIntBetween(1, 10);
+
+        return new ForceMergeStep(stepKey, nextStepKey, null, maxNumSegments);
+    }
+
+    @Override
+    public ForceMergeStep mutateInstance(ForceMergeStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        int maxNumSegments = instance.getMaxNumSegments();
+
+        switch (between(0, 2)) {
+            case 0:
+                key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 2:
+                maxNumSegments += 1;
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new ForceMergeStep(key, nextKey, instance.getClient(), maxNumSegments);
+    }
+
+    @Override
+    public ForceMergeStep copyInstance(ForceMergeStep instance) {
+        return new ForceMergeStep(instance.getKey(), instance.getNextStepKey(),
+            instance.getClient(), instance.getMaxNumSegments());
+    }
+
+    public void testPerformActionComplete() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int maxNumSegments = randomIntBetween(1, 10);
+        Client client = mock(Client.class);
+        AdminClient adminClient = mock(AdminClient.class);
+        IndicesAdminClient indicesClient = mock(IndicesAdminClient.class);
+        when(client.admin()).thenReturn(adminClient);
+        when(adminClient.indices()).thenReturn(indicesClient);
+        ForceMergeResponse forceMergeResponse = Mockito.mock(ForceMergeResponse.class);
+        Mockito.when(forceMergeResponse.getStatus()).thenReturn(RestStatus.OK);
+        Mockito.doAnswer(invocationOnMock -> {
+            ForceMergeRequest request = (ForceMergeRequest) invocationOnMock.getArguments()[0];
+            assertThat(request.maxNumSegments(), equalTo(maxNumSegments));
+            @SuppressWarnings("unchecked")
+            ActionListener<ForceMergeResponse> listener = (ActionListener<ForceMergeResponse>) invocationOnMock.getArguments()[1];
+            listener.onResponse(forceMergeResponse);
+            return null;
+        }).when(indicesClient).forceMerge(any(), any());
+
+        ForceMergeStep step = new ForceMergeStep(stepKey, nextStepKey, client, maxNumSegments);
+        SetOnce<Boolean> completed = new SetOnce<>();
+        step.performAction(indexMetaData, null, new AsyncActionStep.Listener() {
+            @Override
+            public void onResponse(boolean complete) {
+                completed.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("unexpected method call", e);
+            }
+        });
+        assertThat(completed.get(), equalTo(true));
+    }
+
+    public void testPerformActionThrowsException() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException("error");
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int maxNumSegments = randomIntBetween(1, 10);
+        Client client = mock(Client.class);
+        AdminClient adminClient = mock(AdminClient.class);
+        IndicesAdminClient indicesClient = mock(IndicesAdminClient.class);
+        when(client.admin()).thenReturn(adminClient);
+        when(adminClient.indices()).thenReturn(indicesClient);
+        ForceMergeResponse forceMergeResponse = Mockito.mock(ForceMergeResponse.class);
+        Mockito.when(forceMergeResponse.getStatus()).thenReturn(RestStatus.OK);
+        Mockito.doAnswer(invocationOnMock -> {
+            ForceMergeRequest request = (ForceMergeRequest) invocationOnMock.getArguments()[0];
+            assertThat(request.indices().length, equalTo(1));
+            assertThat(request.indices()[0], equalTo(indexMetaData.getIndex().getName()));
+            assertThat(request.maxNumSegments(), equalTo(maxNumSegments));
+            @SuppressWarnings("unchecked")
+            ActionListener<ForceMergeResponse> listener = (ActionListener<ForceMergeResponse>) invocationOnMock.getArguments()[1];
+            listener.onFailure(exception);
+            return null;
+        }).when(indicesClient).forceMerge(any(), any());
+
+        ForceMergeStep step = new ForceMergeStep(stepKey, nextStepKey, client, maxNumSegments);
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, null, new AsyncActionStep.Listener() {
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertEquals(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+        assertThat(exceptionThrown.get(), equalTo(true));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexExplainResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexExplainResponseTests.java
new file mode 100644
index 0000000000000..90915d66e5d47
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexExplainResponseTests.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class IndexExplainResponseTests extends AbstractSerializingTestCase<IndexLifecycleExplainResponse> {
+
+    static IndexLifecycleExplainResponse randomIndexExplainResponse() {
+        if (frequently()) {
+            return randomManagedIndexExplainResponse();
+        } else {
+            return randomUnmanagedIndexExplainResponse();
+        }
+    }
+
+    private static IndexLifecycleExplainResponse randomUnmanagedIndexExplainResponse() {
+        return IndexLifecycleExplainResponse.newUnmanagedIndexResponse(randomAlphaOfLength(10));
+    }
+
+    private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse() {
+        return IndexLifecycleExplainResponse.newManagedIndexResponse(randomAlphaOfLength(10), randomAlphaOfLength(10),
+            randomNonNegativeLong(), randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10),
+            randomBoolean() ? null : randomAlphaOfLength(10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+            randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
+            randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""));
+    }
+
+    @Override
+    protected IndexLifecycleExplainResponse createTestInstance() {
+        return randomIndexExplainResponse();
+    }
+
+    @Override
+    protected Reader<IndexLifecycleExplainResponse> instanceReader() {
+        return IndexLifecycleExplainResponse::new;
+    }
+
+    @Override
+    protected IndexLifecycleExplainResponse doParseInstance(XContentParser parser) throws IOException {
+        return IndexLifecycleExplainResponse.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResponse instance) throws IOException {
+        String index = instance.getIndex();
+        String policy = instance.getPolicyName();
+        String phase = instance.getPhase();
+        String action = instance.getAction();
+        String step = instance.getStep();
+        String failedStep = instance.getFailedStep();
+        Long policyTime = instance.getLifecycleDate();
+        Long phaseTime = instance.getPhaseTime();
+        Long actionTime = instance.getActionTime();
+        Long stepTime = instance.getStepTime();
+        boolean managed = instance.managedByILM();
+        BytesReference stepInfo = instance.getStepInfo();
+        PhaseExecutionInfo phaseExecutionInfo = instance.getPhaseExecutionInfo();
+        if (managed) {
+            switch (between(0, 12)) {
+            case 0:
+                index = index + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 1:
+                policy = policy + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 2:
+                phase = phase + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 3:
+                action = action + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 4:
+                step = step + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 5:
+                if (Strings.hasLength(failedStep) == false) {
+                    failedStep = randomAlphaOfLength(10);
+                } else if (randomBoolean()) {
+                    failedStep = failedStep + randomAlphaOfLengthBetween(1, 5);
+                } else {
+                    failedStep = null;
+                }
+                break;
+            case 6:
+                policyTime += randomLongBetween(0, 100000);
+                break;
+            case 7:
+                phaseTime += randomLongBetween(0, 100000);
+                break;
+            case 8:
+                actionTime += randomLongBetween(0, 100000);
+                break;
+            case 9:
+                stepTime += randomLongBetween(0, 100000);
+                break;
+            case 10:
+                if (Strings.hasLength(stepInfo) == false) {
+                    stepInfo = new BytesArray(randomByteArrayOfLength(100));
+                } else if (randomBoolean()) {
+                    stepInfo = randomValueOtherThan(stepInfo,
+                            () -> new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()));
+                } else {
+                    stepInfo = null;
+                }
+                break;
+            case 11:
+                phaseExecutionInfo = randomValueOtherThan(phaseExecutionInfo, () -> PhaseExecutionInfoTests.randomPhaseExecutionInfo(""));
+                break;
+            case 12:
+                return IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index);
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+            }
+            return IndexLifecycleExplainResponse.newManagedIndexResponse(index, policy, policyTime, phase, action, step, failedStep,
+                    phaseTime, actionTime, stepTime, stepInfo, phaseExecutionInfo);
+        } else {
+            switch (between(0, 1)) {
+            case 0:
+                return IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index + randomAlphaOfLengthBetween(1, 5));
+            case 1:
+                return randomManagedIndexExplainResponse();
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+            }
+        }
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Arrays
+            .asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new)));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME), MockAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+
+    private static class RandomStepInfo implements ToXContentObject {
+
+        private final String key;
+        private final String value;
+
+        RandomStepInfo(Supplier<String> randomStringSupplier) {
+            this.key = randomStringSupplier.get();
+            this.value = randomStringSupplier.get();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(key, value);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(key, value);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            RandomStepInfo other = (RandomStepInfo) obj;
+            return Objects.equals(key, other.key) && Objects.equals(value, other.value);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java
new file mode 100644
index 0000000000000..1035eb7a7462c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class IndexLifecycleFeatureSetUsageTests extends AbstractWireSerializingTestCase<IndexLifecycleFeatureSetUsage> {
+
+    @Override
+    protected IndexLifecycleFeatureSetUsage createTestInstance() {
+        boolean enabled = randomBoolean();
+        boolean available = randomBoolean();
+        List<PolicyStats> policyStats = null;
+        if (enabled) {
+            int size = randomIntBetween(0, 10);
+            policyStats = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                policyStats.add(PolicyStatsTests.createRandomInstance());
+            }
+        }
+        return new IndexLifecycleFeatureSetUsage(available, enabled, policyStats);
+    }
+
+    @Override
+    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) throws IOException {
+        boolean available = instance.available();
+        boolean enabled = instance.enabled();
+        List<PolicyStats> policyStats = instance.getPolicyStats();
+        switch (between(0, 2)) {
+        case 0:
+            available = available == false;
+            break;
+        case 1:
+            enabled = enabled == false;
+            break;
+        case 2:
+            if (policyStats == null) {
+                policyStats = new ArrayList<>();
+                policyStats.add(PolicyStatsTests.createRandomInstance());
+            } else if (randomBoolean()) {
+                policyStats = null;
+            } else {
+                policyStats = new ArrayList<>(policyStats);
+                policyStats.add(PolicyStatsTests.createRandomInstance());
+            }
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new IndexLifecycleFeatureSetUsage(available, enabled, policyStats);
+    }
+
+    @Override
+    protected Reader<IndexLifecycleFeatureSetUsage> instanceReader() {
+        return IndexLifecycleFeatureSetUsage::new;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStepTests.java
new file mode 100644
index 0000000000000..5dbeee07fe75f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStepTests.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.equalTo;
+
+public class InitializePolicyContextStepTests extends AbstractStepTestCase<InitializePolicyContextStep> {
+
+    @Override
+    public InitializePolicyContextStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+
+        return new InitializePolicyContextStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public InitializePolicyContextStep mutateInstance(InitializePolicyContextStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new InitializePolicyContextStep(key, nextKey);
+    }
+
+    @Override
+    public InitializePolicyContextStep copyInstance(InitializePolicyContextStep instance) {
+        return new InitializePolicyContextStep(instance.getKey(), instance.getNextStepKey());
+    }
+
+    public void testAddCreationDate() {
+        long creationDate = randomNonNegativeLong();
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT))
+            .creationDate(creationDate)
+            .numberOfShards(1).numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+        Index index = indexMetadata.getIndex();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        InitializePolicyContextStep step = new InitializePolicyContextStep(null, null);
+        ClusterState newState = step.performAction(index, clusterState);
+        assertThat(getIndexLifecycleDate(index, newState), equalTo(creationDate));
+    }
+
+    public void testDoNothing() {
+        long creationDate = randomNonNegativeLong();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(creationDate);
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .creationDate(creationDate)
+            .numberOfShards(1).numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+        Index index = indexMetadata.getIndex();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        InitializePolicyContextStep step = new InitializePolicyContextStep(null, null);
+        ClusterState newState = step.performAction(index, clusterState);
+        assertTrue(newState == clusterState);
+    }
+
+    private long getIndexLifecycleDate(Index index, ClusterState clusterState) {
+        return LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)).getLifecycleDate();
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionStateTests.java
new file mode 100644
index 0000000000000..e8276ad06ead2
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecycleExecutionStateTests.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class LifecycleExecutionStateTests extends ESTestCase {
+
+    public void testConversion() {
+        Map<String, String> customMetadata = createCustomMetadata();
+        LifecycleExecutionState parsed = LifecycleExecutionState.fromCustomMetadata(customMetadata);
+        assertEquals(customMetadata, parsed.asMap());
+    }
+
+    public void testEmptyValuesAreNotSerialized() {
+        LifecycleExecutionState empty = LifecycleExecutionState.builder().build();
+        assertEquals(new HashMap<String, String>().entrySet(), empty.asMap().entrySet());
+
+        Map<String, String> originalMap = createCustomMetadata();
+        LifecycleExecutionState originalState = LifecycleExecutionState.fromCustomMetadata(originalMap);
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhase(null);
+        assertFalse(newState.build().asMap().containsKey("phase"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setAction(null);
+        assertFalse(newState.build().asMap().containsKey("action"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setStep(null);
+        assertFalse(newState.build().asMap().containsKey("step"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setFailedStep(null);
+        assertFalse(newState.build().asMap().containsKey("failed_step"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhaseDefinition(null);
+        assertFalse(newState.build().asMap().containsKey("phase_definition"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setStepInfo(null);
+        assertFalse(newState.build().asMap().containsKey("step_info"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setPhaseTime(null);
+        assertFalse(newState.build().asMap().containsKey("phase_time"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setActionTime(null);
+        assertFalse(newState.build().asMap().containsKey("action_time"));
+
+        newState = LifecycleExecutionState.builder(originalState);
+        newState.setIndexCreationDate(null);
+        assertFalse(newState.build().asMap().containsKey("creation_date"));
+    }
+
+    public void testEqualsAndHashcode() {
+        LifecycleExecutionState original = LifecycleExecutionState.fromCustomMetadata(createCustomMetadata());
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            original,
+            toCopy -> LifecycleExecutionState.builder(toCopy).build(),
+            LifecycleExecutionStateTests::mutate);
+    }
+
+    private static LifecycleExecutionState mutate(LifecycleExecutionState toMutate) {
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(toMutate);
+        boolean changed = false;
+        if (randomBoolean()) {
+            newState.setPhase(randomValueOtherThan(toMutate.getPhase(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setAction(randomValueOtherThan(toMutate.getAction(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStep(randomValueOtherThan(toMutate.getStep(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setPhaseDefinition(randomValueOtherThan(toMutate.getPhaseDefinition(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setFailedStep(randomValueOtherThan(toMutate.getFailedStep(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStepInfo(randomValueOtherThan(toMutate.getStepInfo(), () -> randomAlphaOfLengthBetween(5, 20)));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setPhaseTime(randomValueOtherThan(toMutate.getPhaseTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setActionTime(randomValueOtherThan(toMutate.getActionTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setStepTime(randomValueOtherThan(toMutate.getStepTime(), ESTestCase::randomLong));
+            changed = true;
+        }
+        if (randomBoolean()) {
+            newState.setIndexCreationDate(randomValueOtherThan(toMutate.getLifecycleDate(), ESTestCase::randomLong));
+            changed = true;
+        }
+
+        if (changed == false) {
+            return LifecycleExecutionState.builder().build();
+        }
+
+        return newState.build();
+    }
+
+    static Map<String, String> createCustomMetadata() {
+        String phase = randomAlphaOfLengthBetween(5,20);
+        String action = randomAlphaOfLengthBetween(5,20);
+        String step = randomAlphaOfLengthBetween(5,20);
+        String failedStep = randomAlphaOfLengthBetween(5,20);
+        String stepInfo = randomAlphaOfLengthBetween(15,50);
+        String phaseDefinition = randomAlphaOfLengthBetween(15,50);
+        long indexCreationDate = randomLong();
+        long phaseTime = randomLong();
+        long actionTime = randomLong();
+        long stepTime = randomLong();
+
+        Map<String, String> customMetadata = new HashMap<>();
+        customMetadata.put("phase", phase);
+        customMetadata.put("action", action);
+        customMetadata.put("step", step);
+        customMetadata.put("failed_step", failedStep);
+        customMetadata.put("step_info", stepInfo);
+        customMetadata.put("phase_definition", phaseDefinition);
+        customMetadata.put("creation_date", String.valueOf(indexCreationDate));
+        customMetadata.put("phase_time", String.valueOf(phaseTime));
+        customMetadata.put("action_time", String.valueOf(actionTime));
+        customMetadata.put("step_time", String.valueOf(stepTime));
+        return customMetadata;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java
new file mode 100644
index 0000000000000..5cb75e132ce92
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyMetadataTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class LifecyclePolicyMetadataTests extends AbstractSerializingTestCase<LifecyclePolicyMetadata> {
+
+    private String lifecycleName;
+
+    @Before
+    public void setup() {
+        lifecycleName = randomAlphaOfLength(20);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Arrays.asList(
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                    (in) -> TimeseriesLifecycleType.INSTANCE),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new)
+            ));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE),
+                (p) -> TimeseriesLifecycleType.INSTANCE),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected LifecyclePolicyMetadata doParseInstance(XContentParser parser) throws IOException {
+        return LifecyclePolicyMetadata.parse(parser, lifecycleName);
+    }
+
+    @Override
+    protected LifecyclePolicyMetadata createTestInstance() {
+        Map<String, String> headers = new HashMap<>();
+        int numberHeaders = between(0, 10);
+        for (int i = 0; i < numberHeaders; i++) {
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        }
+        return new LifecyclePolicyMetadata(LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName), headers,
+            randomNonNegativeLong(), randomNonNegativeLong());
+    }
+
+    @Override
+    protected Reader<LifecyclePolicyMetadata> instanceReader() {
+        return LifecyclePolicyMetadata::new;
+    }
+
+    @Override
+    protected LifecyclePolicyMetadata mutateInstance(LifecyclePolicyMetadata instance) throws IOException {
+        LifecyclePolicy policy = instance.getPolicy();
+        Map<String, String> headers = instance.getHeaders();
+        long version = instance.getVersion();
+        long creationDate = instance.getModifiedDate();
+        switch (between(0, 3)) {
+        case 0:
+            policy = new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, policy.getName() + randomAlphaOfLengthBetween(1, 5),
+                    policy.getPhases());
+            break;
+        case 1:
+            headers = new HashMap<>(headers);
+            headers.put(randomAlphaOfLength(11), randomAlphaOfLength(11));
+            break;
+        case 2:
+            version++;
+            break;
+        case 3:
+            creationDate++;
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new LifecyclePolicyMetadata(policy, headers, version, creationDate);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java
new file mode 100644
index 0000000000000..9d90cc025b0e3
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTests.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+
+public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecyclePolicy> {
+
+    private String lifecycleName;
+
+    @Override
+    protected LifecyclePolicy doParseInstance(XContentParser parser) {
+        return LifecyclePolicy.parse(parser, lifecycleName);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Arrays.asList(
+                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                    (in) -> TimeseriesLifecycleType.INSTANCE),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new)
+            ));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE),
+                (p) -> TimeseriesLifecycleType.INSTANCE),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected LifecyclePolicy createTestInstance() {
+        lifecycleName = randomAlphaOfLength(5);
+        return randomTimeseriesLifecyclePolicy(lifecycleName);
+    }
+
+    /**
+     * The same as {@link #randomTimeseriesLifecyclePolicy(String)} but ensures
+     * that the resulting policy has all valid phases and all valid actions.
+     */
+    public static LifecyclePolicy randomTimeseriesLifecyclePolicyWithAllPhases(@Nullable String lifecycleName) {
+        List<String> phaseNames = TimeseriesLifecycleType.VALID_PHASES;
+        Map<String, Phase> phases = new HashMap<>(phaseNames.size());
+        Function<String, Set<String>> validActions = (phase) ->  {
+            switch (phase) {
+                case "hot":
+                    return TimeseriesLifecycleType.VALID_HOT_ACTIONS;
+                case "warm":
+                    return TimeseriesLifecycleType.VALID_WARM_ACTIONS;
+                case "cold":
+                    return TimeseriesLifecycleType.VALID_COLD_ACTIONS;
+                case "delete":
+                    return TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
+                default:
+                    throw new IllegalArgumentException("invalid phase [" + phase + "]");
+            }};
+        Function<String, LifecycleAction> randomAction = (action) ->  {
+            switch (action) {
+                case AllocateAction.NAME:
+                    return AllocateActionTests.randomInstance();
+                case DeleteAction.NAME:
+                    return new DeleteAction();
+                case ForceMergeAction.NAME:
+                    return ForceMergeActionTests.randomInstance();
+                case ReadOnlyAction.NAME:
+                    return new ReadOnlyAction();
+                case RolloverAction.NAME:
+                    return RolloverActionTests.randomInstance();
+                case ShrinkAction.NAME:
+                    return ShrinkActionTests.randomInstance();
+                default:
+                    throw new IllegalArgumentException("invalid action [" + action + "]");
+            }};
+        for (String phase : phaseNames) {
+            TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            Set<String> actionNames = validActions.apply(phase);
+            for (String action : actionNames) {
+                actions.put(action, randomAction.apply(action));
+            }
+            phases.put(phase, new Phase(phase, after, actions));
+        }
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases);
+    }
+
+    public static LifecyclePolicy randomTimeseriesLifecyclePolicy(@Nullable String lifecycleName) {
+        List<String> phaseNames = randomSubsetOf(TimeseriesLifecycleType.VALID_PHASES);
+        Map<String, Phase> phases = new HashMap<>(phaseNames.size());
+        Function<String, Set<String>> validActions = (phase) ->  {
+            switch (phase) {
+                case "hot":
+                    return TimeseriesLifecycleType.VALID_HOT_ACTIONS;
+                case "warm":
+                    return TimeseriesLifecycleType.VALID_WARM_ACTIONS;
+                case "cold":
+                    return TimeseriesLifecycleType.VALID_COLD_ACTIONS;
+                case "delete":
+                    return TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
+                default:
+                    throw new IllegalArgumentException("invalid phase [" + phase + "]");
+            }};
+        Function<String, LifecycleAction> randomAction = (action) ->  {
+            switch (action) {
+                case AllocateAction.NAME:
+                    return AllocateActionTests.randomInstance();
+                case DeleteAction.NAME:
+                    return new DeleteAction();
+                case ForceMergeAction.NAME:
+                    return ForceMergeActionTests.randomInstance();
+                case ReadOnlyAction.NAME:
+                    return new ReadOnlyAction();
+                case RolloverAction.NAME:
+                    return RolloverActionTests.randomInstance();
+                case ShrinkAction.NAME:
+                    return ShrinkActionTests.randomInstance();
+                default:
+                    throw new IllegalArgumentException("invalid action [" + action + "]");
+            }};
+        for (String phase : phaseNames) {
+            TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            List<String> actionNames = randomSubsetOf(validActions.apply(phase));
+            for (String action : actionNames) {
+                actions.put(action, randomAction.apply(action));
+            }
+            phases.put(phase, new Phase(phase, after, actions));
+        }
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, lifecycleName, phases);
+    }
+
+    public static LifecyclePolicy randomTestLifecyclePolicy(@Nullable String lifecycleName) {
+        int numberPhases = randomInt(5);
+        Map<String, Phase> phases = new HashMap<>(numberPhases);
+        for (int i = 0; i < numberPhases; i++) {
+            TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            if (randomBoolean()) {
+                MockAction action = new MockAction();
+                actions.put(action.getWriteableName(), action);
+            }
+            String phaseName = randomAlphaOfLength(10);
+            phases.put(phaseName, new Phase(phaseName, after, actions));
+        }
+        return new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+    }
+
+    @Override
+    protected LifecyclePolicy mutateInstance(LifecyclePolicy instance) throws IOException {
+        String name = instance.getName();
+        Map<String, Phase> phases = instance.getPhases();
+        switch (between(0, 1)) {
+        case 0:
+            name = name + randomAlphaOfLengthBetween(1, 5);
+            break;
+        case 1:
+            String phaseName = randomValueOtherThanMany(phases::containsKey, () -> randomFrom(TimeseriesLifecycleType.VALID_PHASES));
+            phases = new LinkedHashMap<>(phases);
+            phases.put(phaseName, new Phase(phaseName, TimeValue.timeValueSeconds(randomIntBetween(1, 1000)), Collections.emptyMap()));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new LifecyclePolicy(TimeseriesLifecycleType.INSTANCE, name, phases);
+    }
+
+    @Override
+    protected Reader<LifecyclePolicy> instanceReader() {
+        return LifecyclePolicy::new;
+    }
+
+    public void testFirstAndLastSteps() {
+        Client client = mock(Client.class);
+        lifecycleName = randomAlphaOfLengthBetween(1, 20);
+        Map<String, Phase> phases = new LinkedHashMap<>();
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        List<Step> steps = policy.toSteps(client);
+        assertThat(steps.size(), equalTo(2));
+        assertThat(steps.get(0), instanceOf(InitializePolicyContextStep.class));
+        assertThat(steps.get(0).getKey(), equalTo(new StepKey("new", "init", "init")));
+        assertThat(steps.get(0).getNextStepKey(), equalTo(TerminalPolicyStep.KEY));
+        assertSame(steps.get(1), TerminalPolicyStep.INSTANCE);
+    }
+
+    public void testToStepsWithOneStep() {
+        Client client = mock(Client.class);
+        MockStep mockStep = new MockStep(
+            new Step.StepKey("test", "test", "test"), TerminalPolicyStep.KEY);
+
+        lifecycleName = randomAlphaOfLengthBetween(1, 20);
+        Map<String, Phase> phases = new LinkedHashMap<>();
+        LifecycleAction firstAction = new MockAction(Arrays.asList(mockStep));
+        Map<String, LifecycleAction> actions = Collections.singletonMap(MockAction.NAME, firstAction);
+        Phase firstPhase = new Phase("test", TimeValue.ZERO, actions);
+        phases.put(firstPhase.getName(), firstPhase);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+        StepKey firstStepKey = InitializePolicyContextStep.KEY;
+        StepKey secondStepKey = new StepKey("new", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME);
+        List<Step> steps = policy.toSteps(client);
+        assertThat(steps.size(), equalTo(4));
+        assertSame(steps.get(0).getKey(), firstStepKey);
+        assertThat(steps.get(0).getNextStepKey(), equalTo(secondStepKey));
+        assertThat(steps.get(1).getKey(), equalTo(secondStepKey));
+        assertThat(steps.get(1).getNextStepKey(), equalTo(mockStep.getKey()));
+        assertThat(steps.get(2).getKey(), equalTo(mockStep.getKey()));
+        assertThat(steps.get(2).getNextStepKey(), equalTo(TerminalPolicyStep.KEY));
+        assertSame(steps.get(3), TerminalPolicyStep.INSTANCE);
+    }
+
+    public void testToStepsWithTwoPhases() {
+        Client client = mock(Client.class);
+        MockStep secondActionStep = new MockStep(new StepKey("second_phase", "test2", "test"), TerminalPolicyStep.KEY);
+        MockStep secondAfter = new MockStep(new StepKey("first_phase", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME),
+                secondActionStep.getKey());
+        MockStep firstActionAnotherStep = new MockStep(new StepKey("first_phase", "test", "bar"), secondAfter.getKey());
+        MockStep firstActionStep = new MockStep(new StepKey("first_phase", "test", "foo"), firstActionAnotherStep.getKey());
+        MockStep firstAfter = new MockStep(new StepKey("new", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME), firstActionStep.getKey());
+        MockStep init = new MockStep(InitializePolicyContextStep.KEY, firstAfter.getKey());
+
+        lifecycleName = randomAlphaOfLengthBetween(1, 20);
+        Map<String, Phase> phases = new LinkedHashMap<>();
+        LifecycleAction firstAction = new MockAction(Arrays.asList(firstActionStep, firstActionAnotherStep));
+        LifecycleAction secondAction = new MockAction(Arrays.asList(secondActionStep));
+        Map<String, LifecycleAction> firstActions = Collections.singletonMap(MockAction.NAME, firstAction);
+        Map<String, LifecycleAction> secondActions = Collections.singletonMap(MockAction.NAME, secondAction);
+        Phase firstPhase = new Phase("first_phase", TimeValue.ZERO, firstActions);
+        Phase secondPhase = new Phase("second_phase", TimeValue.ZERO, secondActions);
+        phases.put(firstPhase.getName(), firstPhase);
+        phases.put(secondPhase.getName(), secondPhase);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+
+        List<Step> steps = policy.toSteps(client);
+        assertThat(steps.size(), equalTo(7));
+        assertThat(steps.get(0).getClass(), equalTo(InitializePolicyContextStep.class));
+        assertThat(steps.get(0).getKey(), equalTo(init.getKey()));
+        assertThat(steps.get(0).getNextStepKey(), equalTo(init.getNextStepKey()));
+        assertThat(steps.get(1).getClass(), equalTo(PhaseCompleteStep.class));
+        assertThat(steps.get(1).getKey(), equalTo(firstAfter.getKey()));
+        assertThat(steps.get(1).getNextStepKey(), equalTo(firstAfter.getNextStepKey()));
+        assertThat(steps.get(2), equalTo(firstActionStep));
+        assertThat(steps.get(3), equalTo(firstActionAnotherStep));
+        assertThat(steps.get(4).getClass(), equalTo(PhaseCompleteStep.class));
+        assertThat(steps.get(4).getKey(), equalTo(secondAfter.getKey()));
+        assertThat(steps.get(4).getNextStepKey(), equalTo(secondAfter.getNextStepKey()));
+        assertThat(steps.get(5), equalTo(secondActionStep));
+        assertSame(steps.get(6), TerminalPolicyStep.INSTANCE);
+    }
+
+    public void testIsActionSafe() {
+        Map<String, Phase> phases = new LinkedHashMap<>();
+        LifecycleAction firstAction = new MockAction(Collections.emptyList(), true);
+        LifecycleAction secondAction = new MockAction(Collections.emptyList(), false);
+        Map<String, LifecycleAction> firstActions = Collections.singletonMap(MockAction.NAME, firstAction);
+        Map<String, LifecycleAction> secondActions = Collections.singletonMap(MockAction.NAME, secondAction);
+        Phase firstPhase = new Phase("first_phase", TimeValue.ZERO, firstActions);
+        Phase secondPhase = new Phase("second_phase", TimeValue.ZERO, secondActions);
+        phases.put(firstPhase.getName(), firstPhase);
+        phases.put(secondPhase.getName(), secondPhase);
+        LifecyclePolicy policy = new LifecyclePolicy(TestLifecycleType.INSTANCE, lifecycleName, phases);
+
+        assertTrue(policy.isActionSafe(new StepKey("first_phase", MockAction.NAME, randomAlphaOfLength(10))));
+
+        assertFalse(policy.isActionSafe(new StepKey("second_phase", MockAction.NAME, randomAlphaOfLength(10))));
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> policy.isActionSafe(new StepKey("non_existant_phase", MockAction.NAME, randomAlphaOfLength(10))));
+        assertEquals("Phase [non_existant_phase]  does not exist in policy [" + policy.getName() + "]", exception.getMessage());
+
+        exception = expectThrows(IllegalArgumentException.class,
+                () -> policy.isActionSafe(new StepKey("first_phase", "non_existant_action", randomAlphaOfLength(10))));
+        assertEquals("Action [non_existant_action] in phase [first_phase]  does not exist in policy [" + policy.getName() + "]",
+                exception.getMessage());
+
+        assertTrue(policy.isActionSafe(new StepKey("new", randomAlphaOfLength(10), randomAlphaOfLength(10))));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockAction.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockAction.java
new file mode 100644
index 0000000000000..30eabac562606
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockAction.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class MockAction implements LifecycleAction {
+    public static final String NAME = "TEST_ACTION";
+    private List<Step> steps;
+
+    private static final ObjectParser<MockAction, Void> PARSER = new ObjectParser<>(NAME, MockAction::new);
+    private final boolean safe;
+
+    public static MockAction parse(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public MockAction() {
+        this(Collections.emptyList());
+    }
+
+    public MockAction(List<Step> steps) {
+        this(steps, true);
+    }
+
+    public MockAction(List<Step> steps, boolean safe) {
+        this.steps = steps;
+        this.safe = safe;
+    }
+
+    public MockAction(StreamInput in) throws IOException {
+        this.steps = in.readList(MockStep::new);
+        this.safe = in.readBoolean();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public List<Step> getSteps() {
+        return steps;
+    }
+
+    @Override
+    public boolean isSafeAction() {
+        return safe;
+    }
+
+    @Override
+    public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
+        return new ArrayList<>(steps);
+    }
+
+    @Override
+    public List<StepKey> toStepKeys(String phase) {
+        return steps.stream().map(Step::getKey).collect(Collectors.toList());
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(steps.stream().map(MockStep::new).collect(Collectors.toList()));
+        out.writeBoolean(safe);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(steps, safe);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        MockAction other = (MockAction) obj;
+        return Objects.equals(steps, other.steps) && 
+                Objects.equals(safe, other.safe);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockActionTests.java
new file mode 100644
index 0000000000000..1f10aa051cbe4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockActionTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class MockActionTests extends AbstractActionTestCase<MockAction> {
+
+    @Override
+    protected MockAction createTestInstance() {
+        return new MockAction();
+    }
+
+    @Override
+    protected MockAction doParseInstance(XContentParser parser) throws IOException {
+        return MockAction.parse(parser);
+    }
+
+    @Override
+    protected Reader<MockAction> instanceReader() {
+        return MockAction::new;
+    }
+
+    @Override
+    protected MockAction mutateInstance(MockAction instance) throws IOException {
+        List<Step> steps = instance.getSteps();
+        boolean safe = instance.isSafeAction();
+        if (randomBoolean()) {
+            steps = new ArrayList<>(steps);
+            if (steps.size() > 0) {
+                Step lastStep = steps.remove(steps.size() - 1);
+                if (randomBoolean()) {
+                    Step.StepKey additionalStepKey = randomStepKey();
+                    steps.add(new MockStep(lastStep.getKey(), additionalStepKey));
+                    steps.add(new MockStep(additionalStepKey, null));
+                }
+            } else {
+                steps.add(new MockStep(randomStepKey(), null));
+            }
+        } else {
+            safe = safe == false;
+        }
+        return new MockAction(steps, safe);
+    }
+
+    private static Step.StepKey randomStepKey() {
+        return new Step.StepKey(randomAlphaOfLength(5),
+            randomAlphaOfLength(5), randomAlphaOfLength(5));
+    }
+
+    @Override
+    public void testToSteps() {
+        int numSteps = randomIntBetween(1, 10);
+        List<Step> steps = new ArrayList<>(numSteps);
+        for (int i = 0; i < numSteps; i++) {
+            steps.add(new MockStep(randomStepKey(), randomStepKey()));
+        }
+        MockAction action = new MockAction(steps);
+        assertEquals(action.getSteps(), action.toSteps(null, null, null));
+    }
+}
+
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockStep.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockStep.java
new file mode 100644
index 0000000000000..7de2bd14c5f71
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/MockStep.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+public class MockStep extends Step implements Writeable {
+    public static final String NAME = "TEST_STEP";
+
+    public MockStep(StepKey stepKey, Step.StepKey nextStepKey) {
+        super(stepKey, nextStepKey);
+    }
+
+    public MockStep(Step other) {
+        super(other.getKey(), other.getNextStepKey());
+    }
+
+    public MockStep(StreamInput in) throws IOException {
+        super(new StepKey(in.readString(), in.readString(), in.readString()), readOptionalNextStepKey(in));
+    }
+
+    private static StepKey readOptionalNextStepKey(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            return new StepKey(in.readString(), in.readString(), in.readString());
+        }
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(getKey().getPhase());
+        out.writeString(getKey().getAction());
+        out.writeString(getKey().getName());
+        boolean hasNextStep = getNextStepKey() != null;
+        out.writeBoolean(hasNextStep);
+        if (hasNextStep) {
+            out.writeString(getNextStepKey().getPhase());
+            out.writeString(getNextStepKey().getAction());
+            out.writeString(getNextStepKey().getName());
+        }
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OperationModeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OperationModeTests.java
new file mode 100644
index 0000000000000..d99868fe178a7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/OperationModeTests.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class OperationModeTests extends ESTestCase {
+
+    public void testIsValidChange() {
+        assertFalse(OperationMode.RUNNING.isValidChange(OperationMode.RUNNING));
+        assertTrue(OperationMode.RUNNING.isValidChange(OperationMode.STOPPING));
+        assertFalse(OperationMode.RUNNING.isValidChange(OperationMode.STOPPED));
+
+        assertTrue(OperationMode.STOPPING.isValidChange(OperationMode.RUNNING));
+        assertFalse(OperationMode.STOPPING.isValidChange(OperationMode.STOPPING));
+        assertTrue(OperationMode.STOPPING.isValidChange(OperationMode.STOPPED));
+
+        assertTrue(OperationMode.STOPPED.isValidChange(OperationMode.RUNNING));
+        assertFalse(OperationMode.STOPPED.isValidChange(OperationMode.STOPPING));
+        assertFalse(OperationMode.STOPPED.isValidChange(OperationMode.STOPPED));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStepTests.java
new file mode 100644
index 0000000000000..eea46baadf65f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseCompleteStepTests.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public class PhaseCompleteStepTests extends AbstractStepTestCase<PhaseCompleteStep> {
+
+    @Override
+    public PhaseCompleteStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        return new PhaseCompleteStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public PhaseCompleteStep mutateInstance(PhaseCompleteStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        if (randomBoolean()) {
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        } else {
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        }
+
+        return new PhaseCompleteStep(key, nextKey);
+    }
+
+    @Override
+    public PhaseCompleteStep copyInstance(PhaseCompleteStep instance) {
+        return new PhaseCompleteStep(instance.getKey(), instance.getNextStepKey());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfoTests.java
new file mode 100644
index 0000000000000..9198282a0717c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseExecutionInfoTests.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PhaseExecutionInfoTests extends AbstractSerializingTestCase<PhaseExecutionInfo> {
+
+    static PhaseExecutionInfo randomPhaseExecutionInfo(String phaseName) {
+        return new PhaseExecutionInfo(randomAlphaOfLength(5), PhaseTests.randomTestPhase(phaseName),
+            randomNonNegativeLong(), randomNonNegativeLong());
+    }
+
+    String phaseName;
+
+    @Before
+    public void setupPhaseName() {
+        phaseName = randomAlphaOfLength(7);
+    }
+
+    @Override
+    protected PhaseExecutionInfo createTestInstance() {
+        return randomPhaseExecutionInfo(phaseName);
+    }
+
+    @Override
+    protected Reader<PhaseExecutionInfo> instanceReader() {
+        return PhaseExecutionInfo::new;
+    }
+
+    @Override
+    protected PhaseExecutionInfo doParseInstance(XContentParser parser) throws IOException {
+        return PhaseExecutionInfo.parse(parser, phaseName);
+    }
+
+    @Override
+    protected PhaseExecutionInfo mutateInstance(PhaseExecutionInfo instance) throws IOException {
+        String policyName = instance.getPolicyName();
+        Phase phase = instance.getPhase();
+        long version = instance.getVersion();
+        long modifiedDate = instance.getModifiedDate();
+        switch (between(0, 3)) {
+            case 0:
+                policyName = policyName + randomAlphaOfLengthBetween(1, 5);
+                break;
+            case 1:
+                phase = randomValueOtherThan(phase, () -> PhaseTests.randomTestPhase(randomAlphaOfLength(6)));
+                break;
+            case 2:
+                version++;
+                break;
+            case 3:
+                modifiedDate++;
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+        return new PhaseExecutionInfo(policyName, phase, version, modifiedDate);
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Arrays
+            .asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new)));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME), MockAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseStatsTests.java
new file mode 100644
index 0000000000000..ae4325abe5d87
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseStatsTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class PhaseStatsTests extends AbstractWireSerializingTestCase<PhaseStats> {
+
+    @Override
+    protected PhaseStats createTestInstance() {
+        return createRandomInstance();
+    }
+
+    public static PhaseStats createRandomInstance() {
+        TimeValue after = TimeValue.parseTimeValue(randomTimeValue(), "phase_stats_tests");
+        String[] actionNames = randomArray(0, 20, size -> new String[size], () -> randomAlphaOfLengthBetween(1, 20));
+        return new PhaseStats(after, actionNames);
+    }
+
+    @Override
+    protected PhaseStats mutateInstance(PhaseStats instance) throws IOException {
+        TimeValue after = instance.getAfter();
+        String[] actionNames = instance.getActionNames();
+        switch (between(0, 1)) {
+        case 0:
+            after = randomValueOtherThan(after, () -> TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test"));
+            break;
+        case 1:
+            actionNames = randomValueOtherThanMany(a -> Arrays.equals(a, instance.getActionNames()),
+                    () -> randomArray(0, 20, size -> new String[size], () -> randomAlphaOfLengthBetween(1, 20)));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new PhaseStats(after, actionNames);
+    }
+
+    @Override
+    protected Reader<PhaseStats> instanceReader() {
+        return PhaseStats::new;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseTests.java
new file mode 100644
index 0000000000000..0c3530216f26c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PhaseTests.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PhaseTests extends AbstractSerializingTestCase<Phase> {
+    private String phaseName;
+
+    @Before
+    public void setup() {
+        phaseName = randomAlphaOfLength(20);
+    }
+
+    @Override
+    protected Phase createTestInstance() {
+        return randomTestPhase(phaseName);
+    }
+
+    static Phase randomTestPhase(String phaseName) {
+        TimeValue after = null;
+        if (randomBoolean()) {
+            after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+        }
+        Map<String, LifecycleAction> actions = Collections.emptyMap();
+        if (randomBoolean()) {
+            actions = Collections.singletonMap(MockAction.NAME, new MockAction());
+        }
+        return new Phase(phaseName, after, actions);
+    }
+
+    @Override
+    protected Phase doParseInstance(XContentParser parser) throws IOException {
+        return Phase.parse(parser, phaseName);
+    }
+
+    @Override
+    protected Reader<Phase> instanceReader() {
+        return Phase::new;
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Arrays
+                .asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new)));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.add(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(MockAction.NAME), MockAction::parse));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected Phase mutateInstance(Phase instance) throws IOException {
+        String name = instance.getName();
+        TimeValue after = instance.getMinimumAge();
+        Map<String, LifecycleAction> actions = instance.getActions();
+        switch (between(0, 2)) {
+        case 0:
+            name = name + randomAlphaOfLengthBetween(1, 5);
+            break;
+        case 1:
+            after = TimeValue.timeValueSeconds(after.getSeconds() + randomIntBetween(1, 1000));
+            break;
+        case 2:
+            actions = new HashMap<>(actions);
+            actions.put(MockAction.NAME + "another", new MockAction(Collections.emptyList()));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new Phase(name, after, actions);
+    }
+
+    public void testDefaultAfter() {
+        Phase phase = new Phase(randomAlphaOfLength(20), null, Collections.emptyMap());
+        assertEquals(TimeValue.ZERO, phase.getMinimumAge());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PolicyStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PolicyStatsTests.java
new file mode 100644
index 0000000000000..29a6912a7370f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/PolicyStatsTests.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class PolicyStatsTests extends AbstractWireSerializingTestCase<PolicyStats> {
+
+    @Override
+    protected PolicyStats createTestInstance() {
+        return createRandomInstance();
+    }
+
+    public static PolicyStats createRandomInstance() {
+        int size = randomIntBetween(0, 10);
+        Map<String, PhaseStats> phaseStats = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            phaseStats.put(randomAlphaOfLengthBetween(1, 20), PhaseStatsTests.createRandomInstance());
+        }
+        return new PolicyStats(phaseStats, randomIntBetween(0, 100));
+    }
+
+    @Override
+    protected PolicyStats mutateInstance(PolicyStats instance) throws IOException {
+        Map<String, PhaseStats> phaseStats = instance.getPhaseStats();
+        int indicesManaged = instance.getIndicesManaged();
+        switch (between(0, 1)) {
+        case 0:
+            phaseStats = new HashMap<>(instance.getPhaseStats());
+            phaseStats.put(randomAlphaOfLengthBetween(21, 25), PhaseStatsTests.createRandomInstance());
+            break;
+        case 1:
+            indicesManaged += randomIntBetween(1, 10);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new PolicyStats(phaseStats, indicesManaged);
+    }
+
+    @Override
+    protected Reader<PolicyStats> instanceReader() {
+        return PolicyStats::new;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyActionTests.java
new file mode 100644
index 0000000000000..1d28985fac1db
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ReadOnlyActionTests.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ReadOnlyActionTests extends AbstractActionTestCase<ReadOnlyAction> {
+
+    @Override
+    protected ReadOnlyAction doParseInstance(XContentParser parser) {
+        return ReadOnlyAction.parse(parser);
+    }
+
+    @Override
+    protected ReadOnlyAction createTestInstance() {
+        return new ReadOnlyAction();
+    }
+
+    @Override
+    protected Reader<ReadOnlyAction> instanceReader() {
+        return ReadOnlyAction::new;
+    }
+
+    public void testToSteps() {
+        ReadOnlyAction action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        assertEquals(1, steps.size());
+        StepKey expectedFirstStepKey = new StepKey(phase, ReadOnlyAction.NAME, ReadOnlyAction.NAME);
+        UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
+        assertThat(firstStep.getKey(), equalTo(expectedFirstStepKey));
+        assertThat(firstStep.getNextStepKey(), equalTo(nextStepKey));
+        assertThat(firstStep.getSettings().size(), equalTo(1));
+        assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(firstStep.getSettings()));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java
new file mode 100644
index 0000000000000..f13a09ac7476e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverActionTests.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.List;
+
+public class RolloverActionTests extends AbstractActionTestCase<RolloverAction> {
+
+    @Override
+    protected RolloverAction doParseInstance(XContentParser parser) throws IOException {
+        return RolloverAction.parse(parser);
+    }
+
+    @Override
+    protected RolloverAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static RolloverAction randomInstance() {
+        ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
+        ByteSizeValue maxSize = randomBoolean() ? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
+        Long maxDocs = randomBoolean() ? null : randomNonNegativeLong();
+        TimeValue maxAge = (maxDocs == null && maxSize == null || randomBoolean())
+            ? TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test")
+            : null;
+        return new RolloverAction(maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    protected Reader<RolloverAction> instanceReader() {
+        return RolloverAction::new;
+    }
+
+    @Override
+    protected RolloverAction mutateInstance(RolloverAction instance) throws IOException {
+        ByteSizeValue maxSize = instance.getMaxSize();
+        TimeValue maxAge = instance.getMaxAge();
+        Long maxDocs = instance.getMaxDocs();
+        switch (between(0, 2)) {
+            case 0:
+                maxSize = randomValueOtherThan(maxSize, () -> {
+                    ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
+                    return new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
+                });
+                break;
+            case 1:
+                maxAge = randomValueOtherThan(maxAge,
+                    () -> TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test"));
+                break;
+            case 2:
+                maxDocs = maxDocs == null ? randomNonNegativeLong() : maxDocs + 1;
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+        return new RolloverAction(maxSize, maxAge, maxDocs);
+    }
+
+    public void testNoConditions() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> new RolloverAction(null, null, null));
+        assertEquals("At least one rollover condition must be set.", exception.getMessage());
+    }
+
+    public void testToSteps() {
+        RolloverAction action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+                randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertNotNull(steps);
+        assertEquals(2, steps.size());
+        StepKey expectedFirstStepKey = new StepKey(phase, RolloverAction.NAME, RolloverStep.NAME);
+        StepKey expectedSecondStepKey = new StepKey(phase, RolloverAction.NAME, UpdateRolloverLifecycleDateStep.NAME);
+        RolloverStep firstStep = (RolloverStep) steps.get(0);
+        UpdateRolloverLifecycleDateStep secondStep = (UpdateRolloverLifecycleDateStep) steps.get(1);
+        assertEquals(expectedFirstStepKey, firstStep.getKey());
+        assertEquals(expectedSecondStepKey, secondStep.getKey());
+        assertEquals(secondStep.getKey(), firstStep.getNextStepKey());
+        assertEquals(action.getMaxSize(), firstStep.getMaxSize());
+        assertEquals(action.getMaxAge(), firstStep.getMaxAge());
+        assertEquals(action.getMaxDocs(), firstStep.getMaxDocs());
+        assertEquals(nextStepKey, secondStep.getNextStepKey());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java
new file mode 100644
index 0000000000000..596099e6e275b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/RolloverStepTests.java
@@ -0,0 +1,341 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.rollover.Condition;
+import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
+import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
+import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RolloverStepTests extends AbstractStepTestCase<RolloverStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    public RolloverStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
+        ByteSizeValue maxSize = randomBoolean() ? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
+        Long maxDocs = randomBoolean() ? null : randomNonNegativeLong();
+        TimeValue maxAge = (maxDocs == null && maxSize == null || randomBoolean())
+                ? TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test")
+                : null;
+        return new RolloverStep(stepKey, nextStepKey, client, maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    public RolloverStep mutateInstance(RolloverStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        ByteSizeValue maxSize = instance.getMaxSize();
+        TimeValue maxAge = instance.getMaxAge();
+        Long maxDocs = instance.getMaxDocs();
+
+        switch (between(0, 4)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            maxSize = randomValueOtherThan(maxSize, () -> {
+                ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
+                return new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
+            });
+            break;
+        case 3:
+            maxAge = TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test");
+            break;
+        case 4:
+            maxDocs = randomNonNegativeLong();
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new RolloverStep(key, nextKey, instance.getClient(), maxSize, maxAge, maxDocs);
+    }
+
+    @Override
+    public RolloverStep copyInstance(RolloverStep instance) {
+        return new RolloverStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(),
+            instance.getMaxSize(), instance.getMaxAge(), instance.getMaxDocs());
+    }
+
+    private static void assertRolloverIndexRequest(RolloverRequest request, String alias, Set<Condition<?>> expectedConditions) {
+        assertNotNull(request);
+        assertEquals(1, request.indices().length);
+        assertEquals(alias, request.indices()[0]);
+        assertEquals(alias, request.getAlias());
+        assertEquals(expectedConditions.size(), request.getConditions().size());
+        Set<Object> expectedConditionValues = expectedConditions.stream().map(Condition::value).collect(Collectors.toSet());
+        Set<Object> actualConditionValues = request.getConditions().values().stream()
+            .map(Condition::value).collect(Collectors.toSet());
+        assertEquals(expectedConditionValues, actualConditionValues);
+    }
+
+    public void testPerformAction() {
+        String alias = randomAlphaOfLength(5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .putAlias(AliasMetaData.builder(alias))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        RolloverStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                RolloverRequest request = (RolloverRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
+                Set<Condition<?>> expectedConditions = new HashSet<>();
+                if (step.getMaxAge() != null) {
+                    expectedConditions.add(new MaxAgeCondition(step.getMaxAge()));
+                }
+                if (step.getMaxSize() != null) {
+                    expectedConditions.add(new MaxSizeCondition(step.getMaxSize()));
+                }
+                if (step.getMaxDocs() != null) {
+                    expectedConditions.add(new MaxDocsCondition(step.getMaxDocs()));
+                }
+                assertRolloverIndexRequest(request, alias, expectedConditions);
+                listener.onResponse(new RolloverResponse(null, null, Collections.emptyMap(), request.isDryRun(), true, true, true));
+                return null;
+            }
+
+        }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+
+            @Override
+            public void onResponse(boolean complete, ToXContentObject obj) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(true, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionNotComplete() {
+        String alias = randomAlphaOfLength(5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .putAlias(AliasMetaData.builder(alias))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        RolloverStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                RolloverRequest request = (RolloverRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
+                Set<Condition<?>> expectedConditions = new HashSet<>();
+                if (step.getMaxAge() != null) {
+                    expectedConditions.add(new MaxAgeCondition(step.getMaxAge()));
+                }
+                if (step.getMaxSize() != null) {
+                    expectedConditions.add(new MaxSizeCondition(step.getMaxSize()));
+                }
+                if (step.getMaxDocs() != null) {
+                    expectedConditions.add(new MaxDocsCondition(step.getMaxDocs()));
+                }
+                assertRolloverIndexRequest(request, alias, expectedConditions);
+                listener.onResponse(new RolloverResponse(null, null, Collections.emptyMap(), request.isDryRun(), false, true, true));
+                return null;
+            }
+
+        }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+
+            @Override
+            public void onResponse(boolean complete, ToXContentObject obj) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(false, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionFailure() {
+        String alias = randomAlphaOfLength(5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .putAlias(AliasMetaData.builder(alias))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException();
+        RolloverStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                RolloverRequest request = (RolloverRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
+                Set<Condition<?>> expectedConditions = new HashSet<>();
+                if (step.getMaxAge() != null) {
+                    expectedConditions.add(new MaxAgeCondition(step.getMaxAge()));
+                }
+                if (step.getMaxSize() != null) {
+                    expectedConditions.add(new MaxSizeCondition(step.getMaxSize()));
+                }
+                if (step.getMaxDocs() != null) {
+                    expectedConditions.add(new MaxDocsCondition(step.getMaxDocs()));
+                }
+                assertRolloverIndexRequest(request, alias, expectedConditions);
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).rolloverIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+
+            @Override
+            public void onResponse(boolean complete, ToXContentObject obj) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).rolloverIndex(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionInvalidNullOrEmptyAlias() {
+        String alias = randomBoolean() ? "" : null;
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        RolloverStep step = createRandomInstance();
+
+        SetOnce<Exception> exceptionThrown = new SetOnce<>();
+        step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+            @Override
+            public void onResponse(boolean complete, ToXContentObject obj) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionThrown.set(e);
+            }
+        });
+        assertThat(exceptionThrown.get().getClass(), equalTo(IllegalArgumentException.class));
+        assertThat(exceptionThrown.get().getMessage(), equalTo(String.format(Locale.ROOT,
+            "setting [%s] for index [%s] is empty or not defined", RolloverAction.LIFECYCLE_ROLLOVER_ALIAS,
+            indexMetaData.getIndex().getName())));
+    }
+
+    public void testPerformActionAliasDoesNotPointToIndex() {
+        String alias = randomAlphaOfLength(5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        RolloverStep step = createRandomInstance();
+
+        SetOnce<Exception> exceptionThrown = new SetOnce<>();
+        step.evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+            @Override
+            public void onResponse(boolean complete, ToXContentObject obj) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                exceptionThrown.set(e);
+            }
+        });
+        assertThat(exceptionThrown.get().getClass(), equalTo(IllegalArgumentException.class));
+        assertThat(exceptionThrown.get().getMessage(), equalTo(String.format(Locale.ROOT,
+            "%s [%s] does not point to index [%s]", RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias,
+            indexMetaData.getIndex().getName())));
+
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepInfoTests.java
new file mode 100644
index 0000000000000..a5c1813de2153
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepInfoTests.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.core.indexlifecycle.SegmentCountStep.Info;
+
+import java.io.IOException;
+
+public class SegmentCountStepInfoTests extends AbstractXContentTestCase<SegmentCountStep.Info> {
+
+    @Override
+    protected Info createTestInstance() {
+        return new Info(randomNonNegativeLong());
+    }
+
+    @Override
+    protected Info doParseInstance(XContentParser parser) throws IOException {
+        return Info.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public final void testEqualsAndHashcode() {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+
+    protected final Info copyInstance(Info instance) throws IOException {
+        return new Info(instance.getNumberShardsLeftToMerge());
+    }
+
+    protected Info mutateInstance(Info instance) throws IOException {
+        return createTestInstance();
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java
new file mode 100644
index 0000000000000..ae0551020fbd1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java
@@ -0,0 +1,236 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.segments.IndexSegments;
+import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
+import org.elasticsearch.action.admin.indices.segments.ShardSegments;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.engine.Segment;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+
+public class SegmentCountStepTests extends AbstractStepTestCase<SegmentCountStep> {
+
+    @Override
+    public SegmentCountStep createRandomInstance() {
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int maxNumSegments = randomIntBetween(1, 10);
+
+        return new SegmentCountStep(stepKey, nextStepKey, null, maxNumSegments);
+    }
+
+    private IndexMetaData makeMeta(Index index) {
+        return IndexMetaData.builder(index.getName())
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .build();
+    }
+
+    @Override
+    public SegmentCountStep mutateInstance(SegmentCountStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        int maxNumSegments = instance.getMaxNumSegments();
+
+        switch (between(0, 2)) {
+            case 0:
+                key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 1:
+                nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+                break;
+            case 2:
+                maxNumSegments += 1;
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new SegmentCountStep(key, nextKey, null, maxNumSegments);
+    }
+
+    @Override
+    public SegmentCountStep copyInstance(SegmentCountStep instance) {
+        return new SegmentCountStep(instance.getKey(), instance.getNextStepKey(), null, instance.getMaxNumSegments());
+    }
+
+    public void testIsConditionMet() {
+        int maxNumSegments = randomIntBetween(3, 10);
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Client client = Mockito.mock(Client.class);
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+        IndicesSegmentResponse indicesSegmentResponse = Mockito.mock(IndicesSegmentResponse.class);
+        IndexSegments indexSegments = Mockito.mock(IndexSegments.class);
+        IndexShardSegments indexShardSegments = Mockito.mock(IndexShardSegments.class);
+        Map<Integer, IndexShardSegments> indexShards = Collections.singletonMap(0, indexShardSegments);
+        ShardSegments shardSegmentsOne = Mockito.mock(ShardSegments.class);
+        ShardSegments[] shardSegmentsArray = new ShardSegments[] { shardSegmentsOne };
+        Spliterator<IndexShardSegments> iss = indexShards.values().spliterator();
+        List<Segment> segments = new ArrayList<>();
+        for (int i = 0; i < maxNumSegments - randomIntBetween(0, 3); i++) {
+            segments.add(null);
+        }
+        Mockito.when(indicesSegmentResponse.getStatus()).thenReturn(RestStatus.OK);
+        Mockito.when(indicesSegmentResponse.getIndices()).thenReturn(Collections.singletonMap(index.getName(), indexSegments));
+        Mockito.when(indexSegments.spliterator()).thenReturn(iss);
+        Mockito.when(indexShardSegments.getShards()).thenReturn(shardSegmentsArray);
+        Mockito.when(shardSegmentsOne.getSegments()).thenReturn(segments);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+
+        Mockito.doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<IndicesSegmentResponse> listener = (ActionListener<IndicesSegmentResponse>) invocationOnMock.getArguments()[1];
+            listener.onResponse(indicesSegmentResponse);
+            return null;
+        }).when(indicesClient).segments(any(), any());
+
+        SetOnce<Boolean> conditionMetResult = new SetOnce<>();
+        SetOnce<ToXContentObject> conditionInfo = new SetOnce<>();
+
+        SegmentCountStep step = new SegmentCountStep(stepKey, nextStepKey, client, maxNumSegments);
+        step.evaluateCondition(makeMeta(index), new AsyncWaitStep.Listener() {
+            @Override
+            public void onResponse(boolean conditionMet, ToXContentObject info) {
+                conditionMetResult.set(conditionMet);
+                conditionInfo.set(info);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("unexpected method call");
+            }
+        });
+
+        assertTrue(conditionMetResult.get());
+        assertEquals(new SegmentCountStep.Info(0L), conditionInfo.get());
+    }
+
+    public void testIsConditionFails() {
+        int maxNumSegments = randomIntBetween(3, 10);
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Client client = Mockito.mock(Client.class);
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+        IndicesSegmentResponse indicesSegmentResponse = Mockito.mock(IndicesSegmentResponse.class);
+        IndexSegments indexSegments = Mockito.mock(IndexSegments.class);
+        IndexShardSegments indexShardSegments = Mockito.mock(IndexShardSegments.class);
+        Map<Integer, IndexShardSegments> indexShards = Collections.singletonMap(0, indexShardSegments);
+        ShardSegments shardSegmentsOne = Mockito.mock(ShardSegments.class);
+        ShardSegments[] shardSegmentsArray = new ShardSegments[] { shardSegmentsOne };
+        Spliterator<IndexShardSegments> iss = indexShards.values().spliterator();
+        List<Segment> segments = new ArrayList<>();
+        for (int i = 0; i < maxNumSegments + randomIntBetween(1, 3); i++) {
+            segments.add(null);
+        }
+        Mockito.when(indicesSegmentResponse.getStatus()).thenReturn(RestStatus.OK);
+        Mockito.when(indicesSegmentResponse.getIndices()).thenReturn(Collections.singletonMap(index.getName(), indexSegments));
+        Mockito.when(indexSegments.spliterator()).thenReturn(iss);
+        Mockito.when(indexShardSegments.getShards()).thenReturn(shardSegmentsArray);
+        Mockito.when(shardSegmentsOne.getSegments()).thenReturn(segments);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+
+        Mockito.doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<IndicesSegmentResponse> listener = (ActionListener<IndicesSegmentResponse>) invocationOnMock.getArguments()[1];
+            listener.onResponse(indicesSegmentResponse);
+            return null;
+        }).when(indicesClient).segments(any(), any());
+
+        SetOnce<Boolean> conditionMetResult = new SetOnce<>();
+        SetOnce<ToXContentObject> conditionInfo = new SetOnce<>();
+
+        SegmentCountStep step = new SegmentCountStep(stepKey, nextStepKey, client, maxNumSegments);
+        step.evaluateCondition(makeMeta(index), new AsyncWaitStep.Listener() {
+            @Override
+            public void onResponse(boolean conditionMet, ToXContentObject info) {
+                conditionMetResult.set(conditionMet);
+                conditionInfo.set(info);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("unexpected method call");
+            }
+        });
+
+        assertFalse(conditionMetResult.get());
+        assertEquals(new SegmentCountStep.Info(1L), conditionInfo.get());
+    }
+
+    public void testThrowsException() {
+        Exception exception = new RuntimeException("error");
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        Client client = Mockito.mock(Client.class);
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+
+        Step.StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int maxNumSegments = randomIntBetween(3, 10);
+
+        Mockito.doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<IndicesSegmentResponse> listener = (ActionListener<IndicesSegmentResponse>) invocationOnMock.getArguments()[1];
+            listener.onFailure(exception);
+            return null;
+        }).when(indicesClient).segments(any(), any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+
+        SegmentCountStep step = new SegmentCountStep(stepKey, nextStepKey, client, maxNumSegments);
+        step.evaluateCondition(makeMeta(index), new AsyncWaitStep.Listener() {
+            @Override
+            public void onResponse(boolean conditionMet, ToXContentObject info) {
+                throw new AssertionError("unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertThat(e, equalTo(exception));
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertTrue(exceptionThrown.get());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java
new file mode 100644
index 0000000000000..b42ada6956f87
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java
@@ -0,0 +1,428 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.Settings.Builder;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep.Listener;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SetSingleNodeAllocateStepTests extends AbstractStepTestCase<SetSingleNodeAllocateStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    protected SetSingleNodeAllocateStep createRandomInstance() {
+        return new SetSingleNodeAllocateStep(randomStepKey(), randomStepKey(), client);
+    }
+
+    @Override
+    protected SetSingleNodeAllocateStep mutateInstance(SetSingleNodeAllocateStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        switch (between(0, 1)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new SetSingleNodeAllocateStep(key, nextKey, instance.getClient());
+    }
+
+    @Override
+    protected SetSingleNodeAllocateStep copyInstance(SetSingleNodeAllocateStep instance) {
+        return new SetSingleNodeAllocateStep(instance.getKey(), instance.getNextStepKey(), client);
+    }
+
+    public static void assertSettingsRequestContainsValueFrom(UpdateSettingsRequest request, String settingsKey,
+                                                              Set<String> acceptableValues, boolean assertOnlyKeyInSettings,
+                                                              String... expectedIndices) {
+        assertNotNull(request);
+        assertArrayEquals(expectedIndices, request.indices());
+        assertThat(request.settings().get(settingsKey), anyOf(acceptableValues.stream().map(e -> equalTo(e)).collect(Collectors.toList())));
+        if (assertOnlyKeyInSettings) {
+            assertEquals(1, request.settings().size());
+        }
+    }
+
+    public void testPerformActionNoAttrs() throws IOException {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Set<String> validNodeIds = new HashSet<>();
+        Settings validNodeSettings = Settings.EMPTY;
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName).build();
+            nodes.add(
+                    DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
+            validNodeIds.add(nodeId);
+        }
+
+        assertNodeSelected(indexMetaData, index, validNodeIds, nodes);
+    }
+
+    public void testPerformActionAttrsAllNodesValid() throws IOException {
+        int numAttrs = randomIntBetween(1, 10);
+        String[][] validAttrs = new String[numAttrs][2];
+        for (int i = 0; i < numAttrs; i++) {
+            validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
+        }
+        Settings.Builder indexSettings = settings(Version.CURRENT);
+        for (String[] attr : validAttrs) {
+            indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
+        }
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Set<String> validNodeIds = new HashSet<>();
+        Settings validNodeSettings = Settings.EMPTY;
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            String[] nodeAttr = randomFrom(validAttrs);
+            Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                    .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr[0], nodeAttr[1]).build();
+            nodes.add(DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
+            validNodeIds.add(nodeId);
+        }
+
+        assertNodeSelected(indexMetaData, index, validNodeIds, nodes);
+    }
+
+    public void testPerformActionAttrsSomeNodesValid() throws IOException {
+        String[] validAttr = new String[] { "box_type", "valid" };
+        String[] invalidAttr = new String[] { "box_type", "not_valid" };
+        Settings.Builder indexSettings = settings(Version.CURRENT);
+        indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + validAttr[0], validAttr[1]);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Set<String> validNodeIds = new HashSet<>();
+        Settings validNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + validAttr[0], validAttr[1]).build();
+        Settings invalidNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + invalidAttr[0], invalidAttr[1]).build();
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            Builder nodeSettingsBuilder = Settings.builder();
+            // randomise whether the node had valid attributes or not but make sure at least one node is valid
+            if (randomBoolean() || (i == numNodes - 1 && validNodeIds.isEmpty())) {
+                nodeSettingsBuilder.put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName);
+                validNodeIds.add(nodeId);
+            } else {
+                nodeSettingsBuilder.put(invalidNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName);
+            }
+            nodes.add(DiscoveryNode.createLocal(nodeSettingsBuilder.build(), new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                    nodeId));
+        }
+
+        assertNodeSelected(indexMetaData, index, validNodeIds, nodes);
+    }
+
+    public void testPerformActionAttrsNoNodesValid() {
+        String[] validAttr = new String[] { "box_type", "valid" };
+        String[] invalidAttr = new String[] { "box_type", "not_valid" };
+        Settings.Builder indexSettings = settings(Version.CURRENT);
+        indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + validAttr[0], validAttr[1]);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Settings invalidNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + invalidAttr[0], invalidAttr[1]).build();
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            Builder nodeSettingsBuilder = Settings.builder().put(invalidNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName);
+            nodes.add(DiscoveryNode.createLocal(nodeSettingsBuilder.build(), new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                    nodeId));
+        }
+
+        assertNoValidNode(indexMetaData, index, nodes);
+    }
+
+    public void testPerformActionAttrsRequestFails() {
+        int numAttrs = randomIntBetween(1, 10);
+        String[][] validAttrs = new String[numAttrs][2];
+        for (int i = 0; i < numAttrs; i++) {
+            validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
+        }
+        Settings.Builder indexSettings = settings(Version.CURRENT);
+        for (String[] attr : validAttrs) {
+            indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
+        }
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Set<String> validNodeIds = new HashSet<>();
+        Settings validNodeSettings = Settings.EMPTY;
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            String[] nodeAttr = randomFrom(validAttrs);
+            Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                    .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr[0], nodeAttr[1]).build();
+            nodes.add(DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
+            validNodeIds.add(nodeId);
+        }
+
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+                indexMetaData);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+
+        SetSingleNodeAllocateStep step = createRandomInstance();
+        Exception exception = new RuntimeException();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertSettingsRequestContainsValueFrom(request,
+                        IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
+                        indexMetaData.getIndex().getName());
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, clusterState, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionAttrsNoShard() {
+        int numAttrs = randomIntBetween(1, 10);
+        String[][] validAttrs = new String[numAttrs][2];
+        for (int i = 0; i < numAttrs; i++) {
+            validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
+        }
+        Settings.Builder indexSettings = settings(Version.CURRENT);
+        for (String[] attr : validAttrs) {
+            indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
+        }
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Index index = indexMetaData.getIndex();
+        Set<String> validNodeIds = new HashSet<>();
+        Settings validNodeSettings = Settings.EMPTY;
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        int numNodes = randomIntBetween(1, 20);
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "node_id_" + i;
+            String nodeName = "node_" + i;
+            int nodePort = 9300 + i;
+            String[] nodeAttr = randomFrom(validAttrs);
+            Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                    .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr[0], nodeAttr[1]).build();
+            nodes.add(DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
+            validNodeIds.add(nodeId);
+        }
+
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+                indexMetaData);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index);
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+
+        SetSingleNodeAllocateStep step = createRandomInstance();
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, clusterState, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertThat(e, Matchers.instanceOf(IndexNotFoundException.class));
+                assertEquals(indexMetaData.getIndex(), ((IndexNotFoundException) e).getIndex());
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verifyZeroInteractions(client);
+    }
+
+    private void assertNodeSelected(IndexMetaData indexMetaData, Index index,
+                                    Set<String> validNodeIds, DiscoveryNodes.Builder nodes) throws IOException {
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+                indexMetaData);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+
+        SetSingleNodeAllocateStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertSettingsRequestContainsValueFrom(request,
+                        IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
+                        indexMetaData.getIndex().getName());
+                listener.onResponse(new AcknowledgedResponse(true));
+                return null;
+            }
+
+        }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+
+        step.performAction(indexMetaData, clusterState, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(true, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any());
+    }
+
+    private void assertNoValidNode(IndexMetaData indexMetaData, Index index, DiscoveryNodes.Builder nodes) {
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
+                indexMetaData);
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
+                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
+                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+
+        SetSingleNodeAllocateStep step = createRandomInstance();
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+
+        step.performAction(indexMetaData, clusterState, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(false, actionCompleted.get());
+
+        Mockito.verifyZeroInteractions(client);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java
new file mode 100644
index 0000000000000..658f8bef6d47b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
+
+    @Override
+    protected ShrinkAction doParseInstance(XContentParser parser) throws IOException {
+        return ShrinkAction.parse(parser);
+    }
+
+    @Override
+    protected ShrinkAction createTestInstance() {
+        return randomInstance();
+    }
+
+    static ShrinkAction randomInstance() {
+        return new ShrinkAction(randomIntBetween(1, 100));
+    }
+
+    @Override
+    protected ShrinkAction mutateInstance(ShrinkAction action) {
+        return new ShrinkAction(action.getNumberOfShards() + randomIntBetween(1, 2));
+    }
+
+    @Override
+    protected Reader<ShrinkAction> instanceReader() {
+        return ShrinkAction::new;
+    }
+
+    public void testNonPositiveShardNumber() {
+        Exception e = expectThrows(Exception.class, () -> new ShrinkAction(randomIntBetween(-100, 0)));
+        assertThat(e.getMessage(), equalTo("[number_of_shards] must be greater than 0"));
+    }
+
+    public void testToSteps() {
+        ShrinkAction action = createTestInstance();
+        String phase = randomAlphaOfLengthBetween(1, 10);
+        StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
+            randomAlphaOfLengthBetween(1, 10));
+        List<Step> steps = action.toSteps(null, phase, nextStepKey);
+        assertThat(steps.size(), equalTo(8));
+        StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
+        StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
+        StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
+        StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
+        StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
+        StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
+        StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
+        StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
+
+        assertTrue(steps.get(0) instanceof UpdateSettingsStep);
+        assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey));
+        assertThat(steps.get(0).getNextStepKey(), equalTo(expectedSecondKey));
+        assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(0)).getSettings()));
+
+        assertTrue(steps.get(1) instanceof SetSingleNodeAllocateStep);
+        assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey));
+        assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey));
+
+        assertTrue(steps.get(2) instanceof CheckShrinkReadyStep);
+        assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey));
+        assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey));
+
+        assertTrue(steps.get(3) instanceof ShrinkStep);
+        assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));
+        assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey));
+        assertThat(((ShrinkStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+
+        assertTrue(steps.get(4) instanceof ShrunkShardsAllocatedStep);
+        assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey));
+        assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey));
+        assertThat(((ShrunkShardsAllocatedStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+
+        assertTrue(steps.get(5) instanceof CopyExecutionStateStep);
+        assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey));
+        assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey));
+        assertThat(((CopyExecutionStateStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+
+        assertTrue(steps.get(6) instanceof ShrinkSetAliasStep);
+        assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey));
+        assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey));
+        assertThat(((ShrinkSetAliasStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+
+        assertTrue(steps.get(7) instanceof ShrunkenIndexCheckStep);
+        assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey));
+        assertThat(steps.get(7).getNextStepKey(), equalTo(nextStepKey));
+        assertThat(((ShrunkenIndexCheckStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
+    }
+
+    @Override
+    protected boolean isSafeAction() {
+        return false;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStepTests.java
new file mode 100644
index 0000000000000..5fcfcdeea38f0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkSetAliasStepTests.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep.Listener;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShrinkSetAliasStepTests extends AbstractStepTestCase<ShrinkSetAliasStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    public ShrinkSetAliasStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new ShrinkSetAliasStep(stepKey, nextStepKey, client, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrinkSetAliasStep mutateInstance(ShrinkSetAliasStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+        switch (between(0, 2)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            shrunkIndexPrefix += randomAlphaOfLength(5);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new ShrinkSetAliasStep(key, nextKey, instance.getClient(), shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrinkSetAliasStep copyInstance(ShrinkSetAliasStep instance) {
+        return new ShrinkSetAliasStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getShrunkIndexPrefix());
+    }
+
+    public void testPerformAction() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ShrinkSetAliasStep step = createRandomInstance();
+
+        String sourceIndex = indexMetaData.getIndex().getName();
+        String shrunkenIndex = step.getShrunkIndexPrefix() + sourceIndex;
+        List<AliasActions> expectedAliasActions = Arrays.asList(
+            IndicesAliasesRequest.AliasActions.removeIndex().index(sourceIndex),
+            IndicesAliasesRequest.AliasActions.add().index(shrunkenIndex).alias(sourceIndex));
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                IndicesAliasesRequest request = (IndicesAliasesRequest) invocation.getArguments()[0];
+                assertThat(request.getAliasActions(), equalTo(expectedAliasActions));
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                listener.onResponse(new AcknowledgedResponse(true));
+                return null;
+            }
+
+        }).when(indicesClient).aliases(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertTrue(actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).aliases(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionFailure() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException();
+        ShrinkSetAliasStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).aliases(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).aliases(Mockito.any(), Mockito.any());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStepTests.java
new file mode 100644
index 0000000000000..472c22025e195
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkStepTests.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep.Listener;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShrinkStepTests extends AbstractStepTestCase<ShrinkStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    public ShrinkStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        int numberOfShards = randomIntBetween(1, 20);
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new ShrinkStep(stepKey, nextStepKey, client, numberOfShards, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrinkStep mutateInstance(ShrinkStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        int numberOfShards = instance.getNumberOfShards();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+
+        switch (between(0, 3)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            numberOfShards = numberOfShards + 1;
+            break;
+        case 3:
+            shrunkIndexPrefix += randomAlphaOfLength(5);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new ShrinkStep(key, nextKey, instance.getClient(), numberOfShards, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrinkStep copyInstance(ShrinkStep instance) {
+        return new ShrinkStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getNumberOfShards(),
+                instance.getShrunkIndexPrefix());
+    }
+
+    public void testPerformAction() throws Exception {
+        String lifecycleName = randomAlphaOfLength(5);
+        ShrinkStep step = createRandomInstance();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(step.getKey().getPhase());
+        lifecycleState.setAction(step.getKey().getAction());
+        lifecycleState.setStep(step.getKey().getName());
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
+        IndexMetaData sourceIndexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)
+            )
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5))
+            .putAlias(AliasMetaData.builder("my_alias"))
+            .build();
+
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                ResizeRequest request = (ResizeRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<ResizeResponse> listener = (ActionListener<ResizeResponse>) invocation.getArguments()[1];
+                assertThat(request.getSourceIndex(), equalTo(sourceIndexMetaData.getIndex().getName()));
+                assertThat(request.getTargetIndexRequest().aliases(), equalTo(Collections.singleton(new Alias("my_alias"))));
+                assertThat(request.getTargetIndexRequest().settings(), equalTo(Settings.builder()
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, step.getNumberOfShards())
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, sourceIndexMetaData.getNumberOfReplicas())
+                    .put(LifecycleSettings.LIFECYCLE_NAME, lifecycleName)
+                    .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", (String) null)
+                    .build()));
+                assertThat(request.getTargetIndexRequest().settings()
+                        .getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), equalTo(step.getNumberOfShards()));
+                listener.onResponse(new ResizeResponse(true, true, sourceIndexMetaData.getIndex().getName()));
+                return null;
+            }
+
+        }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.performAction(sourceIndexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(true, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionNotComplete() throws Exception {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ShrinkStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                @SuppressWarnings("unchecked")
+                ActionListener<ResizeResponse> listener = (ActionListener<ResizeResponse>) invocation.getArguments()[1];
+                listener.onResponse(new ResizeResponse(false, false, indexMetaData.getIndex().getName()));
+                return null;
+            }
+
+        }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(false, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionFailure() throws Exception {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(randomNonNegativeLong());
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException();
+        ShrinkStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                @SuppressWarnings("unchecked")
+                ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArguments()[1];
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepInfoTests.java
new file mode 100644
index 0000000000000..1ff0e04531302
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepInfoTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrunkShardsAllocatedStep.Info;
+
+import java.io.IOException;
+
+public class ShrunkShardsAllocatedStepInfoTests extends AbstractXContentTestCase<ShrunkShardsAllocatedStep.Info> {
+
+    @Override
+    protected Info createTestInstance() {
+        return new Info(randomBoolean(), randomIntBetween(0, 10000), randomBoolean());
+    }
+
+    @Override
+    protected Info doParseInstance(XContentParser parser) throws IOException {
+        return Info.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public final void testEqualsAndHashcode() {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+
+    protected final Info copyInstance(Info instance) throws IOException {
+        return new Info(instance.shrunkIndexExists(), instance.getActualShards(), instance.allShardsActive());
+    }
+
+    protected Info mutateInstance(Info instance) throws IOException {
+        boolean shrunkIndexExists = instance.shrunkIndexExists();
+        int actualShards = instance.getActualShards();
+        boolean allShardsActive = instance.allShardsActive();
+        switch (between(0, 2)) {
+        case 0:
+            shrunkIndexExists = shrunkIndexExists == false;
+            break;
+        case 1:
+            actualShards += between(1, 20);
+            break;
+        case 2:
+            allShardsActive = allShardsActive == false;
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new Info(shrunkIndexExists, actualShards, allShardsActive);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepTests.java
new file mode 100644
index 0000000000000..272b50499d7fd
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkShardsAllocatedStepTests.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep.Result;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public class ShrunkShardsAllocatedStepTests extends AbstractStepTestCase<ShrunkShardsAllocatedStep> {
+
+    @Override
+    public ShrunkShardsAllocatedStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new ShrunkShardsAllocatedStep(stepKey, nextStepKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrunkShardsAllocatedStep mutateInstance(ShrunkShardsAllocatedStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+
+        switch (between(0, 2)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            shrunkIndexPrefix += randomAlphaOfLength(5);
+            break;
+        default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new ShrunkShardsAllocatedStep(key, nextKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrunkShardsAllocatedStep copyInstance(ShrunkShardsAllocatedStep instance) {
+        return new ShrunkShardsAllocatedStep(instance.getKey(), instance.getNextStepKey(), instance.getShrunkIndexPrefix());
+    }
+
+    public void testConditionMet() {
+        ShrunkShardsAllocatedStep step = createRandomInstance();
+        int shrinkNumberOfShards = randomIntBetween(1, 5);
+        int originalNumberOfShards = randomIntBetween(1, 5);
+        String originalIndexName = randomAlphaOfLength(5);
+        IndexMetaData originalIndexMetadata = IndexMetaData.builder(originalIndexName)
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(originalNumberOfShards)
+            .numberOfReplicas(0).build();
+        IndexMetaData shrunkIndexMetadata = IndexMetaData.builder(step.getShrunkIndexPrefix() + originalIndexName)
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(shrinkNumberOfShards)
+                .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(originalIndexMetadata))
+            .put(IndexMetaData.builder(shrunkIndexMetadata))
+            .build();
+        Index shrinkIndex = shrunkIndexMetadata.getIndex();
+
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+
+        IndexRoutingTable.Builder builder = IndexRoutingTable.builder(shrinkIndex);
+        for (int i = 0; i < shrinkNumberOfShards; i++) {
+            builder.addShard(TestShardRouting.newShardRouting(new ShardId(shrinkIndex, i),
+                nodeId, true, ShardRoutingState.STARTED));
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .routingTable(RoutingTable.builder().add(builder.build()).build()).build();
+
+        Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState);
+        assertTrue(result.isComplete());
+        assertNull(result.getInfomationContext());
+    }
+
+    public void testConditionNotMetBecauseOfActive() {
+        ShrunkShardsAllocatedStep step = createRandomInstance();
+        int shrinkNumberOfShards = randomIntBetween(1, 5);
+        int originalNumberOfShards = randomIntBetween(1, 5);
+        String originalIndexName = randomAlphaOfLength(5);
+        IndexMetaData originalIndexMetadata = IndexMetaData.builder(originalIndexName)
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(originalNumberOfShards)
+            .numberOfReplicas(0).build();
+        IndexMetaData shrunkIndexMetadata = IndexMetaData.builder(step.getShrunkIndexPrefix() + originalIndexName)
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(shrinkNumberOfShards)
+                .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(originalIndexMetadata))
+            .put(IndexMetaData.builder(shrunkIndexMetadata))
+            .build();
+        Index shrinkIndex = shrunkIndexMetadata.getIndex();
+
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+
+        IndexRoutingTable.Builder builder = IndexRoutingTable.builder(shrinkIndex);
+        for (int i = 0; i < shrinkNumberOfShards; i++) {
+            builder.addShard(TestShardRouting.newShardRouting(new ShardId(shrinkIndex, i),
+                nodeId, true, ShardRoutingState.INITIALIZING));
+        }
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .routingTable(RoutingTable.builder().add(builder.build()).build()).build();
+
+        Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState);
+        assertFalse(result.isComplete());
+        assertEquals(new ShrunkShardsAllocatedStep.Info(true, shrinkNumberOfShards, false),
+                result.getInfomationContext());
+    }
+
+    public void testConditionNotMetBecauseOfShrunkIndexDoesntExistYet() {
+        ShrunkShardsAllocatedStep step = createRandomInstance();
+        int originalNumberOfShards = randomIntBetween(1, 5);
+        String originalIndexName = randomAlphaOfLength(5);
+        IndexMetaData originalIndexMetadata = IndexMetaData.builder(originalIndexName)
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(originalNumberOfShards)
+            .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(originalIndexMetadata))
+            .build();
+
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        Result result = step.isConditionMet(originalIndexMetadata.getIndex(), clusterState);
+        assertFalse(result.isComplete());
+        assertEquals(new ShrunkShardsAllocatedStep.Info(false, -1, false), result.getInfomationContext());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepInfoTests.java
new file mode 100644
index 0000000000000..b4a9021af214e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepInfoTests.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrunkenIndexCheckStep.Info;
+
+import java.io.IOException;
+
+public class ShrunkenIndexCheckStepInfoTests extends AbstractXContentTestCase<ShrunkenIndexCheckStep.Info> {
+
+    @Override
+    protected Info createTestInstance() {
+        return new Info(randomAlphaOfLengthBetween(10, 20));
+    }
+
+    @Override
+    protected Info doParseInstance(XContentParser parser) throws IOException {
+        return Info.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public final void testEqualsAndHashcode() {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copyInstance, this::mutateInstance);
+        }
+    }
+
+    protected final Info copyInstance(Info instance) throws IOException {
+        return new Info(instance.getOriginalIndexName());
+    }
+
+    protected Info mutateInstance(Info instance) throws IOException {
+        return new Info(randomValueOtherThan(instance.getOriginalIndexName(), () -> randomAlphaOfLengthBetween(10, 20)));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepTests.java
new file mode 100644
index 0000000000000..b7268ca9b259f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrunkenIndexCheckStepTests.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep.Result;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ShrunkenIndexCheckStepTests extends AbstractStepTestCase<ShrunkenIndexCheckStep> {
+
+    @Override
+    public ShrunkenIndexCheckStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        String shrunkIndexPrefix = randomAlphaOfLength(10);
+        return new ShrunkenIndexCheckStep(stepKey, nextStepKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrunkenIndexCheckStep mutateInstance(ShrunkenIndexCheckStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        String shrunkIndexPrefix = instance.getShrunkIndexPrefix();
+        switch (between(0, 2)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            shrunkIndexPrefix += randomAlphaOfLength(5);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new ShrunkenIndexCheckStep(key, nextKey, shrunkIndexPrefix);
+    }
+
+    @Override
+    public ShrunkenIndexCheckStep copyInstance(ShrunkenIndexCheckStep instance) {
+        return new ShrunkenIndexCheckStep(instance.getKey(), instance.getNextStepKey(), instance.getShrunkIndexPrefix());
+    }
+
+    public void testConditionMet() {
+        ShrunkenIndexCheckStep step = createRandomInstance();
+        String sourceIndex = randomAlphaOfLengthBetween(1, 10);
+        IndexMetaData indexMetadata = IndexMetaData.builder(step.getShrunkIndexPrefix() + sourceIndex)
+            .settings(settings(Version.CURRENT).put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY, sourceIndex))
+            .numberOfShards(1)
+            .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
+        assertTrue(result.isComplete());
+        assertNull(result.getInfomationContext());
+    }
+
+    public void testConditionNotMetBecauseNotSameShrunkenIndex() {
+        ShrunkenIndexCheckStep step = createRandomInstance();
+        String sourceIndex = randomAlphaOfLengthBetween(1, 10);
+        IndexMetaData shrinkIndexMetadata = IndexMetaData.builder(sourceIndex + "hello")
+            .settings(settings(Version.CURRENT).put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY, sourceIndex))
+            .numberOfShards(1)
+            .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(shrinkIndexMetadata))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        Result result = step.isConditionMet(shrinkIndexMetadata.getIndex(), clusterState);
+        assertFalse(result.isComplete());
+        assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.getInfomationContext());
+    }
+
+    public void testConditionNotMetBecauseSourceIndexExists() {
+        ShrunkenIndexCheckStep step = createRandomInstance();
+        String sourceIndex = randomAlphaOfLengthBetween(1, 10);
+        IndexMetaData originalIndexMetadata = IndexMetaData.builder(sourceIndex)
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(100)
+            .numberOfReplicas(0).build();
+        IndexMetaData shrinkIndexMetadata = IndexMetaData.builder(step.getShrunkIndexPrefix() + sourceIndex)
+            .settings(settings(Version.CURRENT).put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY, sourceIndex))
+            .numberOfShards(1)
+            .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(originalIndexMetadata))
+            .put(IndexMetaData.builder(shrinkIndexMetadata))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        Result result = step.isConditionMet(shrinkIndexMetadata.getIndex(), clusterState);
+        assertFalse(result.isComplete());
+        assertEquals(new ShrunkenIndexCheckStep.Info(sourceIndex), result.getInfomationContext());
+    }
+
+    public void testIllegalState() {
+        ShrunkenIndexCheckStep step = createRandomInstance();
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0).build();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        IllegalStateException exception = expectThrows(IllegalStateException.class,
+            () -> step.isConditionMet(indexMetadata.getIndex(), clusterState));
+        assertThat(exception.getMessage(),
+            equalTo("step[is-shrunken-index] is checking an un-shrunken index[" + indexMetadata.getIndex().getName() + "]"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequestTests.java
new file mode 100644
index 0000000000000..4c61f3016a13e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StartILMRequestTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+public class StartILMRequestTests extends AbstractStreamableTestCase<StartILMRequest> {
+
+    @Override
+    protected StartILMRequest createBlankInstance() {
+        return new StartILMRequest();
+    }
+
+    @Override
+    protected StartILMRequest createTestInstance() {
+        return new StartILMRequest();
+    }
+
+    public void testValidate() {
+        StartILMRequest request = createTestInstance();
+        assertNull(request.validate());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StepKeyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StepKeyTests.java
new file mode 100644
index 0000000000000..ae90a150b7c4c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StepKeyTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public class StepKeyTests extends AbstractSerializingTestCase<StepKey> {
+
+    @Override
+    public StepKey createTestInstance() {
+        return new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected Writeable.Reader<StepKey> instanceReader() {
+        return StepKey::new;
+    }
+
+    @Override
+    protected StepKey doParseInstance(XContentParser parser) {
+        return StepKey.parse(parser);
+    }
+
+    @Override
+    public StepKey mutateInstance(StepKey instance) {
+        String phase = instance.getPhase();
+        String action = instance.getAction();
+        String step = instance.getName();
+
+        switch (between(0, 2)) {
+        case 0:
+            phase += randomAlphaOfLength(5);
+            break;
+        case 1:
+            action += randomAlphaOfLength(5);
+            break;
+        case 2:
+            step += randomAlphaOfLength(5);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new StepKey(phase, action, step);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequestTests.java
new file mode 100644
index 0000000000000..be603ee33acc1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/StopILMRequestTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+public class StopILMRequestTests extends AbstractStreamableTestCase<StopILMRequest> {
+
+    @Override
+    protected StopILMRequest createBlankInstance() {
+        return new StopILMRequest();
+    }
+
+    @Override
+    protected StopILMRequest createTestInstance() {
+        return new StopILMRequest();
+    }
+
+    public void testValidate() {
+        StopILMRequest request = createTestInstance();
+        assertNull(request.validate());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStepTests.java
new file mode 100644
index 0000000000000..1db1523b5cd82
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TerminalPolicyStepTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+public class TerminalPolicyStepTests extends AbstractStepTestCase<TerminalPolicyStep> {
+
+    @Override
+    public TerminalPolicyStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        return new TerminalPolicyStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public TerminalPolicyStep mutateInstance(TerminalPolicyStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        if (randomBoolean()) {
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        } else {
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        }
+
+        return new TerminalPolicyStep(key, nextKey);
+    }
+
+    @Override
+    public TerminalPolicyStep copyInstance(TerminalPolicyStep instance) {
+        return new TerminalPolicyStep(instance.getKey(), instance.getNextStepKey());
+    }
+    public void testInstance() {
+        assertEquals(new Step.StepKey("completed", "completed", "completed"), TerminalPolicyStep.INSTANCE.getKey());
+        assertNull(TerminalPolicyStep.INSTANCE.getNextStepKey());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TestLifecycleType.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TestLifecycleType.java
new file mode 100644
index 0000000000000..f68798e9331d1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TestLifecycleType.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class TestLifecycleType implements LifecycleType {
+    public static final TestLifecycleType INSTANCE = new TestLifecycleType();
+
+    public static final String TYPE = "test";
+
+    private TestLifecycleType() {
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public void validate(Collection<Phase> phases) {
+        // always valid
+    }
+
+    @Override
+    public List<Phase> getOrderedPhases(Map<String, Phase> phases) {
+        return new ArrayList<>(phases.values());
+    }
+
+    @Override
+    public String getNextPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        List<String> orderedPhaseNames = getOrderedPhases(phases).stream().map(Phase::getName).collect(Collectors.toList());
+        int index = orderedPhaseNames.indexOf(currentPhaseName);
+        if (index < 0) {
+            throw new IllegalArgumentException("[" + currentPhaseName + "] is not a valid phase for lifecycle type [" + TYPE + "]");
+        } else if (index == orderedPhaseNames.size() - 1) {
+            return null;
+        } else {
+            return orderedPhaseNames.get(index + 1);
+        }
+    }
+
+    @Override
+    public String getPreviousPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        List<String> orderedPhaseNames = getOrderedPhases(phases).stream().map(Phase::getName).collect(Collectors.toList());
+        int index = orderedPhaseNames.indexOf(currentPhaseName);
+        if (index < 0) {
+            throw new IllegalArgumentException("[" + currentPhaseName + "] is not a valid phase for lifecycle type [" + TYPE + "]");
+        } else if (index == 0) {
+            return null;
+        } else {
+            return orderedPhaseNames.get(index - 1);
+        }
+    }
+
+    @Override
+    public List<LifecycleAction> getOrderedActions(Phase phase) {
+        return new ArrayList<>(phase.getActions().values());
+    }
+
+    @Override
+    public String getNextActionName(String currentActionName, Phase phase) {
+        List<String> orderedActionNames = getOrderedActions(phase).stream().map(LifecycleAction::getWriteableName)
+                .collect(Collectors.toList());
+        int index = orderedActionNames.indexOf(currentActionName);
+        if (index < 0) {
+            throw new IllegalArgumentException("[" + currentActionName + "] is not a valid action for phase [" + phase.getName()
+                    + "] in lifecycle type [" + TYPE + "]");
+        } else if (index == orderedActionNames.size() - 1) {
+            return null;
+        } else {
+            return orderedActionNames.get(index + 1);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java
new file mode 100644
index 0000000000000..2f0c2f8d18b33
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleTypeTests.java
@@ -0,0 +1,483 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.ORDERED_VALID_COLD_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.ORDERED_VALID_DELETE_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.ORDERED_VALID_HOT_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.ORDERED_VALID_WARM_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.VALID_COLD_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.VALID_HOT_ACTIONS;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.VALID_PHASES;
+import static org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType.VALID_WARM_ACTIONS;
+import static org.hamcrest.Matchers.equalTo;
+
+public class TimeseriesLifecycleTypeTests extends ESTestCase {
+
+    private static final AllocateAction TEST_ALLOCATE_ACTION =
+        new AllocateAction(2, Collections.singletonMap("node", "node1"),null, null);
+    private static final DeleteAction TEST_DELETE_ACTION = new DeleteAction();
+    private static final ForceMergeAction TEST_FORCE_MERGE_ACTION = new ForceMergeAction(1);
+    private static final RolloverAction TEST_ROLLOVER_ACTION = new RolloverAction(new ByteSizeValue(1), null, null);
+    private static final ShrinkAction TEST_SHRINK_ACTION = new ShrinkAction(1);
+    private static final ReadOnlyAction TEST_READ_ONLY_ACTION = new ReadOnlyAction();
+
+    public void testValidatePhases() {
+        boolean invalid = randomBoolean();
+        String phaseName = randomFrom("hot", "warm", "cold", "delete");
+        if (invalid) {
+            phaseName += randomAlphaOfLength(5);
+        }
+        Map<String, Phase> phases = Collections.singletonMap(phaseName,
+            new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap()));
+        if (invalid) {
+            Exception e = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.INSTANCE.validate(phases.values()));
+            assertThat(e.getMessage(), equalTo("Timeseries lifecycle does not support phase [" + phaseName + "]"));
+        } else {
+            TimeseriesLifecycleType.INSTANCE.validate(phases.values());
+        }
+    }
+
+    public void testValidateHotPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = VALID_HOT_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("allocate", "forcemerge", "delete", "shrink"));
+            actions.put(invalidAction.getWriteableName(), invalidAction);
+        }
+        Map<String, Phase> hotPhase = Collections.singletonMap("hot",
+            new Phase("hot", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                    () -> TimeseriesLifecycleType.INSTANCE.validate(hotPhase.values()));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getWriteableName() + "] defined in phase [hot]"));
+        } else {
+            TimeseriesLifecycleType.INSTANCE.validate(hotPhase.values());
+        }
+    }
+
+    public void testValidateWarmPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = randomSubsetOf(VALID_WARM_ACTIONS)
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("rollover", "delete"));
+            actions.put(invalidAction.getWriteableName(), invalidAction);
+        }
+        Map<String, Phase> warmPhase = Collections.singletonMap("warm",
+            new Phase("warm", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                    () -> TimeseriesLifecycleType.INSTANCE.validate(warmPhase.values()));
+            assertThat(e.getMessage(),
+            equalTo("invalid action [" + invalidAction.getWriteableName() + "] defined in phase [warm]"));
+        } else {
+            TimeseriesLifecycleType.INSTANCE.validate(warmPhase.values());
+        }
+    }
+
+    public void testValidateColdPhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = randomSubsetOf(VALID_COLD_ACTIONS)
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("rollover", "delete", "forcemerge", "shrink"));
+            actions.put(invalidAction.getWriteableName(), invalidAction);
+        }
+        Map<String, Phase> coldPhase = Collections.singletonMap("cold",
+            new Phase("cold", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                    () -> TimeseriesLifecycleType.INSTANCE.validate(coldPhase.values()));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getWriteableName() + "] defined in phase [cold]"));
+        } else {
+            TimeseriesLifecycleType.INSTANCE.validate(coldPhase.values());
+        }
+    }
+
+    public void testValidateDeletePhase() {
+        LifecycleAction invalidAction = null;
+        Map<String, LifecycleAction> actions = VALID_DELETE_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        if (randomBoolean()) {
+            invalidAction = getTestAction(randomFrom("allocate", "rollover", "forcemerge", "shrink"));
+            actions.put(invalidAction.getWriteableName(), invalidAction);
+        }
+        Map<String, Phase> deletePhase = Collections.singletonMap("delete",
+            new Phase("delete", TimeValue.ZERO, actions));
+
+        if (invalidAction != null) {
+            Exception e = expectThrows(IllegalArgumentException.class,
+                    () -> TimeseriesLifecycleType.INSTANCE.validate(deletePhase.values()));
+            assertThat(e.getMessage(),
+                equalTo("invalid action [" + invalidAction.getWriteableName() + "] defined in phase [delete]"));
+        } else {
+            TimeseriesLifecycleType.INSTANCE.validate(deletePhase.values());
+        }
+    }
+
+    public void testGetOrderedPhases() {
+        Map<String, Phase> phaseMap = new HashMap<>();
+        for (String phaseName : randomSubsetOf(randomIntBetween(0, VALID_PHASES.size()), VALID_PHASES)) {
+            phaseMap.put(phaseName, new Phase(phaseName, TimeValue.ZERO, Collections.emptyMap()));
+        }
+
+
+        assertTrue(isSorted(TimeseriesLifecycleType.INSTANCE.getOrderedPhases(phaseMap), Phase::getName, VALID_PHASES));
+    }
+
+
+    public void testGetOrderedActionsInvalidPhase() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.INSTANCE
+            .getOrderedActions(new Phase("invalid", TimeValue.ZERO, Collections.emptyMap())));
+        assertThat(exception.getMessage(), equalTo("lifecycle type[timeseries] does not support phase[invalid]"));
+    }
+
+    public void testGetOrderedActionsHot() {
+        Map<String, LifecycleAction> actions = VALID_HOT_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        Phase hotPhase = new Phase("hot", TimeValue.ZERO, actions);
+        List<LifecycleAction> orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(hotPhase);
+        assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_HOT_ACTIONS));
+    }
+
+    public void testGetOrderedActionsWarm() {
+        Map<String, LifecycleAction> actions = VALID_WARM_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        Phase warmPhase = new Phase("warm", TimeValue.ZERO, actions);
+        List<LifecycleAction> orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(warmPhase);
+        assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_WARM_ACTIONS));
+    }
+
+    public void testGetOrderedActionsCold() {
+        Map<String, LifecycleAction> actions = VALID_COLD_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        Phase coldPhase = new Phase("cold", TimeValue.ZERO, actions);
+        List<LifecycleAction> orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(coldPhase);
+        assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_COLD_ACTIONS));
+    }
+
+    public void testGetOrderedActionsDelete() {
+        Map<String, LifecycleAction> actions = VALID_DELETE_ACTIONS
+            .stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
+        Phase deletePhase = new Phase("delete", TimeValue.ZERO, actions);
+        List<LifecycleAction> orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(deletePhase);
+        assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_DELETE_ACTIONS));
+    }
+
+    public void testGetNextPhaseName() {
+        assertNextPhaseName("hot", "warm", new String[] { "hot", "warm" });
+        assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold" });
+        assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold", "delete" });
+        assertNextPhaseName("hot", "warm", new String[] { "warm", "cold", "delete" });
+        assertNextPhaseName("hot", "warm", new String[] { "warm", "cold", "delete" });
+        assertNextPhaseName("hot", "warm", new String[] { "warm", "delete" });
+        assertNextPhaseName("hot", "cold", new String[] { "cold", "delete" });
+        assertNextPhaseName("hot", "cold", new String[] { "cold" });
+        assertNextPhaseName("hot", "delete", new String[] { "hot", "delete" });
+        assertNextPhaseName("hot", "delete", new String[] { "delete" });
+        assertNextPhaseName("hot", null, new String[] { "hot" });
+        assertNextPhaseName("hot", null, new String[] {});
+
+        assertNextPhaseName("warm", "cold", new String[] { "hot", "warm", "cold", "delete" });
+        assertNextPhaseName("warm", "cold", new String[] { "warm", "cold", "delete" });
+        assertNextPhaseName("warm", "cold", new String[] { "cold", "delete" });
+        assertNextPhaseName("warm", "cold", new String[] { "cold" });
+        assertNextPhaseName("warm", "delete", new String[] { "hot", "warm", "delete" });
+        assertNextPhaseName("warm", null, new String[] { "hot", "warm" });
+        assertNextPhaseName("warm", null, new String[] { "warm" });
+        assertNextPhaseName("warm", null, new String[] { "hot" });
+        assertNextPhaseName("warm", null, new String[] {});
+
+        assertNextPhaseName("cold", "delete", new String[] { "hot", "warm", "cold", "delete" });
+        assertNextPhaseName("cold", "delete", new String[] { "warm", "cold", "delete" });
+        assertNextPhaseName("cold", "delete", new String[] { "cold", "delete" });
+        assertNextPhaseName("cold", "delete", new String[] { "delete" });
+        assertNextPhaseName("cold", "delete", new String[] { "hot", "warm", "delete" });
+        assertNextPhaseName("cold", null, new String[] { "hot", "warm", "cold" });
+        assertNextPhaseName("cold", null, new String[] { "hot", "warm" });
+        assertNextPhaseName("cold", null, new String[] { "cold" });
+        assertNextPhaseName("cold", null, new String[] { "hot" });
+        assertNextPhaseName("cold", null, new String[] {});
+
+        assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold" });
+        assertNextPhaseName("delete", null, new String[] { "hot", "warm" });
+        assertNextPhaseName("delete", null, new String[] { "cold" });
+        assertNextPhaseName("delete", null, new String[] { "hot" });
+        assertNextPhaseName("delete", null, new String[] {});
+        assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold", "delete" });
+        assertNextPhaseName("delete", null, new String[] { "hot", "warm", "delete" });
+        assertNextPhaseName("delete", null, new String[] { "cold", "delete" });
+        assertNextPhaseName("delete", null, new String[] { "delete" });
+        assertNextPhaseName("delete", null, new String[] {});
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> TimeseriesLifecycleType.INSTANCE.getNextPhaseName("foo", Collections.emptyMap()));
+        assertEquals("[foo] is not a valid phase for lifecycle type [" + TimeseriesLifecycleType.TYPE + "]", exception.getMessage());
+        exception = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.INSTANCE
+                .getNextPhaseName("foo", Collections.singletonMap("foo", new Phase("foo", TimeValue.ZERO, Collections.emptyMap()))));
+        assertEquals("[foo] is not a valid phase for lifecycle type [" + TimeseriesLifecycleType.TYPE + "]", exception.getMessage());
+    }
+
+    public void testGetPreviousPhaseName() {
+        assertPreviousPhaseName("hot", null, new String[] { "hot", "warm" });
+        assertPreviousPhaseName("hot", null, new String[] { "hot", "warm", "cold" });
+        assertPreviousPhaseName("hot", null, new String[] { "hot", "warm", "cold", "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "warm", "cold", "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "warm", "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "cold", "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "cold" });
+        assertPreviousPhaseName("hot", null, new String[] { "hot", "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "delete" });
+        assertPreviousPhaseName("hot", null, new String[] { "hot" });
+        assertPreviousPhaseName("hot", null, new String[] {});
+
+        assertPreviousPhaseName("warm", "hot", new String[] { "hot", "warm", "cold", "delete" });
+        assertPreviousPhaseName("warm", null, new String[] { "warm", "cold", "delete" });
+        assertPreviousPhaseName("warm", "hot", new String[] { "hot", "cold", "delete" });
+        assertPreviousPhaseName("warm", null, new String[] { "cold", "delete" });
+        assertPreviousPhaseName("warm", "hot", new String[] { "hot", "delete" });
+        assertPreviousPhaseName("warm", null, new String[] { "delete" });
+        assertPreviousPhaseName("warm", "hot", new String[] { "hot" });
+        assertPreviousPhaseName("warm", null, new String[] {});
+
+        assertPreviousPhaseName("cold", "warm", new String[] { "hot", "warm", "cold", "delete" });
+        assertPreviousPhaseName("cold", "hot", new String[] { "hot", "cold", "delete" });
+        assertPreviousPhaseName("cold", "warm", new String[] { "warm", "cold", "delete" });
+        assertPreviousPhaseName("cold", null, new String[] { "cold", "delete" });
+        assertPreviousPhaseName("cold", "warm", new String[] { "hot", "warm", "delete" });
+        assertPreviousPhaseName("cold", "hot", new String[] { "hot", "delete" });
+        assertPreviousPhaseName("cold", "warm", new String[] { "warm", "delete" });
+        assertPreviousPhaseName("cold", null, new String[] { "delete" });
+        assertPreviousPhaseName("cold", "warm", new String[] { "hot", "warm" });
+        assertPreviousPhaseName("cold", "hot", new String[] { "hot" });
+        assertPreviousPhaseName("cold", "warm", new String[] { "warm" });
+        assertPreviousPhaseName("cold", null, new String[] {});
+
+        assertPreviousPhaseName("delete", "cold", new String[] { "hot", "warm", "cold", "delete" });
+        assertPreviousPhaseName("delete", "cold", new String[] { "warm", "cold", "delete" });
+        assertPreviousPhaseName("delete", "warm", new String[] { "hot", "warm", "delete" });
+        assertPreviousPhaseName("delete", "hot", new String[] { "hot", "delete" });
+        assertPreviousPhaseName("delete", "cold", new String[] { "cold", "delete" });
+        assertPreviousPhaseName("delete", null, new String[] { "delete" });
+        assertPreviousPhaseName("delete", "cold", new String[] { "hot", "warm", "cold" });
+        assertPreviousPhaseName("delete", "cold", new String[] { "warm", "cold" });
+        assertPreviousPhaseName("delete", "warm", new String[] { "hot", "warm" });
+        assertPreviousPhaseName("delete", "hot", new String[] { "hot" });
+        assertPreviousPhaseName("delete", "cold", new String[] { "cold" });
+        assertPreviousPhaseName("delete", null, new String[] {});
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> TimeseriesLifecycleType.INSTANCE.getPreviousPhaseName("foo", Collections.emptyMap()));
+        assertEquals("[foo] is not a valid phase for lifecycle type [" + TimeseriesLifecycleType.TYPE + "]", exception.getMessage());
+        exception = expectThrows(IllegalArgumentException.class, () -> TimeseriesLifecycleType.INSTANCE
+                .getPreviousPhaseName("foo", Collections.singletonMap("foo", new Phase("foo", TimeValue.ZERO, Collections.emptyMap()))));
+        assertEquals("[foo] is not a valid phase for lifecycle type [" + TimeseriesLifecycleType.TYPE + "]", exception.getMessage());
+    }
+
+    public void testGetNextActionName() {
+        // Hot Phase
+        assertNextActionName("hot", RolloverAction.NAME, null, new String[] {});
+        assertNextActionName("hot", RolloverAction.NAME, null, new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", "foo", new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", AllocateAction.NAME, new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", DeleteAction.NAME, new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", ForceMergeAction.NAME, new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", ReadOnlyAction.NAME, new String[] { RolloverAction.NAME });
+        assertInvalidAction("hot", ShrinkAction.NAME, new String[] { RolloverAction.NAME });
+
+        // Warm Phase
+        assertNextActionName("warm", ReadOnlyAction.NAME, AllocateAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, ShrinkAction.NAME,
+                new String[] { ReadOnlyAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, ForceMergeAction.NAME,
+                new String[] { ReadOnlyAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, null, new String[] { ReadOnlyAction.NAME });
+
+        assertNextActionName("warm", ReadOnlyAction.NAME, AllocateAction.NAME,
+                new String[] { AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, ShrinkAction.NAME, new String[] { ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME });
+        assertNextActionName("warm", ReadOnlyAction.NAME, null, new String[] {});
+
+        assertNextActionName("warm", AllocateAction.NAME, ShrinkAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", AllocateAction.NAME, ForceMergeAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", AllocateAction.NAME, null, new String[] { ReadOnlyAction.NAME, AllocateAction.NAME });
+
+        assertNextActionName("warm", AllocateAction.NAME, ShrinkAction.NAME, new String[] { ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", AllocateAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME });
+        assertNextActionName("warm", AllocateAction.NAME, null, new String[] {});
+
+        assertNextActionName("warm", ShrinkAction.NAME, ForceMergeAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertNextActionName("warm", ShrinkAction.NAME, null,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME });
+
+        assertNextActionName("warm", ShrinkAction.NAME, ForceMergeAction.NAME, new String[] { ForceMergeAction.NAME });
+        assertNextActionName("warm", ShrinkAction.NAME, null, new String[] {});
+
+        assertNextActionName("warm", ForceMergeAction.NAME, null,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+
+        assertNextActionName("warm", ForceMergeAction.NAME, null, new String[] {});
+
+        assertInvalidAction("warm", "foo", new String[] { RolloverAction.NAME });
+        assertInvalidAction("warm", DeleteAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+        assertInvalidAction("warm", RolloverAction.NAME,
+                new String[] { ReadOnlyAction.NAME, AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME });
+
+        // Cold Phase
+        assertNextActionName("cold", AllocateAction.NAME, null, new String[] { AllocateAction.NAME });
+
+        assertNextActionName("cold", AllocateAction.NAME, null, new String[] {});
+
+        assertNextActionName("cold", AllocateAction.NAME, null, new String[] {});
+
+        assertInvalidAction("cold", "foo", new String[] { AllocateAction.NAME });
+        assertInvalidAction("cold", DeleteAction.NAME, new String[] { AllocateAction.NAME });
+        assertInvalidAction("cold", ForceMergeAction.NAME, new String[] { AllocateAction.NAME });
+        assertInvalidAction("cold", ReadOnlyAction.NAME, new String[] { AllocateAction.NAME });
+        assertInvalidAction("cold", RolloverAction.NAME, new String[] { AllocateAction.NAME });
+        assertInvalidAction("cold", ShrinkAction.NAME, new String[] { AllocateAction.NAME });
+
+        // Delete Phase
+        assertNextActionName("delete", DeleteAction.NAME, null, new String[] {});
+        assertNextActionName("delete", DeleteAction.NAME, null, new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", "foo", new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", AllocateAction.NAME, new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", ForceMergeAction.NAME, new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", ReadOnlyAction.NAME, new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", RolloverAction.NAME, new String[] { DeleteAction.NAME });
+        assertInvalidAction("delete", ShrinkAction.NAME, new String[] { DeleteAction.NAME });
+
+        Phase phase = new Phase("foo", TimeValue.ZERO, Collections.emptyMap());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> TimeseriesLifecycleType.INSTANCE.getNextActionName(ShrinkAction.NAME, phase));
+        assertEquals("lifecycle type[" + TimeseriesLifecycleType.TYPE + "] does not support phase[" + phase.getName() + "]",
+                exception.getMessage());
+    }
+
+    private void assertNextActionName(String phaseName, String currentAction, String expectedNextAction, String... availableActionNames) {
+        Map<String, LifecycleAction> availableActions = convertActionNamesToActions(availableActionNames);
+        Phase phase = new Phase(phaseName, TimeValue.ZERO, availableActions);
+        String nextAction = TimeseriesLifecycleType.INSTANCE.getNextActionName(currentAction, phase);
+        assertEquals(expectedNextAction, nextAction);
+    }
+
+    private void assertInvalidAction(String phaseName, String currentAction, String... availableActionNames) {
+        Map<String, LifecycleAction> availableActions = convertActionNamesToActions(availableActionNames);
+        Phase phase = new Phase(phaseName, TimeValue.ZERO, availableActions);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> TimeseriesLifecycleType.INSTANCE.getNextActionName(currentAction, phase));
+        assertEquals("[" + currentAction + "] is not a valid action for phase [" + phaseName + "] in lifecycle type ["
+                + TimeseriesLifecycleType.TYPE + "]", exception.getMessage());
+    }
+
+    private ConcurrentMap<String, LifecycleAction> convertActionNamesToActions(String... availableActionNames) {
+        return Arrays.asList(availableActionNames).stream().map(n -> {
+            switch (n) {
+            case AllocateAction.NAME:
+                return new AllocateAction(null, Collections.singletonMap("foo", "bar"), Collections.emptyMap(), Collections.emptyMap());
+            case DeleteAction.NAME:
+                return new DeleteAction();
+            case ForceMergeAction.NAME:
+                return new ForceMergeAction(1);
+            case ReadOnlyAction.NAME:
+                return new ReadOnlyAction();
+            case RolloverAction.NAME:
+                return new RolloverAction(ByteSizeValue.parseBytesSizeValue("0b", "test"), TimeValue.ZERO, 1L);
+            case ShrinkAction.NAME:
+                return new ShrinkAction(1);
+            }
+            return new DeleteAction();
+        }).collect(Collectors.toConcurrentMap(LifecycleAction::getWriteableName, Function.identity()));
+    }
+
+    private void assertNextPhaseName(String currentPhase, String expectedNextPhase, String... availablePhaseNames) {
+        Map<String, Phase> availablePhases = Arrays.asList(availablePhaseNames).stream()
+                .map(n -> new Phase(n, TimeValue.ZERO, Collections.emptyMap()))
+                .collect(Collectors.toMap(Phase::getName, Function.identity()));
+        String nextPhase = TimeseriesLifecycleType.INSTANCE.getNextPhaseName(currentPhase, availablePhases);
+        assertEquals(expectedNextPhase, nextPhase);
+    }
+
+    private void assertPreviousPhaseName(String currentPhase, String expectedNextPhase, String... availablePhaseNames) {
+        Map<String, Phase> availablePhases = Arrays.asList(availablePhaseNames).stream()
+                .map(n -> new Phase(n, TimeValue.ZERO, Collections.emptyMap()))
+                .collect(Collectors.toMap(Phase::getName, Function.identity()));
+        String nextPhase = TimeseriesLifecycleType.INSTANCE.getPreviousPhaseName(currentPhase, availablePhases);
+        assertEquals(expectedNextPhase, nextPhase);
+    }
+
+    /**
+     * checks whether an ordered list of objects (usually Phase and LifecycleAction) are found in the same
+     * order as the ordered VALID_PHASES/VALID_HOT_ACTIONS/... lists
+     * @param orderedObjects the ordered objects to verify sort order of
+     * @param getKey the way to retrieve the key to sort against (Phase#getName, LifecycleAction#getName)
+     * @param validOrderedKeys the source of truth of the sort order
+     * @param <T> the type of object
+     */
+    private <T> boolean isSorted(List<T> orderedObjects, Function<T, String> getKey, List<String> validOrderedKeys) {
+        int validIndex = 0;
+        for (T obj : orderedObjects) {
+            String key = getKey.apply(obj);
+            int i = validIndex;
+            for (; i < validOrderedKeys.size(); i++) {
+                if (validOrderedKeys.get(i).equals(key)) {
+                    validIndex = i;
+                    break;
+                }
+            }
+            if (i == validOrderedKeys.size()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private LifecycleAction getTestAction(String actionName) {
+        switch (actionName) {
+            case AllocateAction.NAME:
+                return TEST_ALLOCATE_ACTION;
+            case DeleteAction.NAME:
+                return TEST_DELETE_ACTION;
+            case ForceMergeAction.NAME:
+                return TEST_FORCE_MERGE_ACTION;
+            case ReadOnlyAction.NAME:
+                return TEST_READ_ONLY_ACTION;
+            case RolloverAction.NAME:
+                return TEST_ROLLOVER_ACTION;
+            case ShrinkAction.NAME:
+                return TEST_SHRINK_ACTION;
+            default:
+                throw new IllegalArgumentException("unsupported timeseries phase action [" + actionName + "]");
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java
new file mode 100644
index 0000000000000..6e492e24f9b33
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateRolloverLifecycleDateStepTests.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase<UpdateRolloverLifecycleDateStep> {
+
+    @Override
+    public UpdateRolloverLifecycleDateStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        return new UpdateRolloverLifecycleDateStep(stepKey, nextStepKey);
+    }
+
+    @Override
+    public UpdateRolloverLifecycleDateStep mutateInstance(UpdateRolloverLifecycleDateStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+
+        if (randomBoolean()) {
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        } else {
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+        }
+
+        return new UpdateRolloverLifecycleDateStep(key, nextKey);
+    }
+
+    @Override
+    public UpdateRolloverLifecycleDateStep copyInstance(UpdateRolloverLifecycleDateStep instance) {
+        return new UpdateRolloverLifecycleDateStep(instance.getKey(), instance.getNextStepKey());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testPerformAction() {
+        String alias = randomAlphaOfLength(3);
+        long creationDate = randomLongBetween(0, 1000000);
+        long rolloverTime = randomValueOtherThan(creationDate, () -> randomNonNegativeLong());
+        IndexMetaData newIndexMetaData = IndexMetaData.builder(randomAlphaOfLength(11))
+            .settings(settings(Version.CURRENT)).creationDate(creationDate)
+            .putAlias(AliasMetaData.builder(alias)).numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5)).build();
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .putRolloverInfo(new RolloverInfo(alias, Collections.emptyList(), rolloverTime))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder()
+                .put(indexMetaData, false)
+                .put(newIndexMetaData, false)).build();
+
+        UpdateRolloverLifecycleDateStep step = createRandomInstance();
+        ClusterState newState = step.performAction(indexMetaData.getIndex(), clusterState);
+        long actualRolloverTime = LifecycleExecutionState
+            .fromIndexMetadata(newState.metaData().index(indexMetaData.getIndex()))
+            .getLifecycleDate();
+        assertThat(actualRolloverTime, equalTo(rolloverTime));
+    }
+
+    public void testPerformActionBeforeRolloverHappened() {
+        String alias = randomAlphaOfLength(3);
+        long creationDate = randomLongBetween(0, 1000000);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(11))
+            .settings(settings(Version.CURRENT).put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias))
+            .creationDate(creationDate).putAlias(AliasMetaData.builder(alias)).numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder().put(indexMetaData, false)).build();
+        UpdateRolloverLifecycleDateStep step = createRandomInstance();
+
+        IllegalStateException exceptionThrown = expectThrows(IllegalStateException.class,
+            () -> step.performAction(indexMetaData.getIndex(), clusterState));
+        assertThat(exceptionThrown.getMessage(),
+            equalTo("no rollover info found for [" + indexMetaData.getIndex().getName() + "], either the index " +
+                "has not yet rolled over or a subsequent index was created outside of Index Lifecycle Management"));
+    }
+
+    public void testPerformActionWithNoRolloverAliasSetting() {
+        long creationDate = randomLongBetween(0, 1000000);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(11))
+            .settings(settings(Version.CURRENT)).creationDate(creationDate).numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(MetaData.builder().put(indexMetaData, false)).build();
+        UpdateRolloverLifecycleDateStep step = createRandomInstance();
+
+        IllegalStateException exceptionThrown = expectThrows(IllegalStateException.class,
+            () -> step.performAction(indexMetaData.getIndex(), clusterState));
+        assertThat(exceptionThrown.getMessage(),
+            equalTo("setting [index.lifecycle.rollover_alias] is not set on index [" + indexMetaData.getIndex().getName() +"]"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStepTests.java
new file mode 100644
index 0000000000000..22908146af21f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/UpdateSettingsStepTests.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep.Listener;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class UpdateSettingsStepTests extends AbstractStepTestCase<UpdateSettingsStep> {
+
+    private Client client;
+
+    @Before
+    public void setup() {
+        client = Mockito.mock(Client.class);
+    }
+
+    @Override
+    public UpdateSettingsStep createRandomInstance() {
+        StepKey stepKey = randomStepKey();
+        StepKey nextStepKey = randomStepKey();
+        Settings settings = Settings.builder().put(randomAlphaOfLength(10), randomAlphaOfLength(10)).build();
+
+        return new UpdateSettingsStep(stepKey, nextStepKey, client, settings);
+    }
+
+    @Override
+    public UpdateSettingsStep mutateInstance(UpdateSettingsStep instance) {
+        StepKey key = instance.getKey();
+        StepKey nextKey = instance.getNextStepKey();
+        Settings settings = instance.getSettings();
+
+        switch (between(0, 2)) {
+        case 0:
+            key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 1:
+            nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
+            break;
+        case 2:
+            settings = Settings.builder().put(settings).put(randomAlphaOfLength(10), randomInt()).build();
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new UpdateSettingsStep(key, nextKey, client, settings);
+    }
+
+    @Override
+    public UpdateSettingsStep copyInstance(UpdateSettingsStep instance) {
+        return new UpdateSettingsStep(instance.getKey(), instance.getNextStepKey(), instance.getClient(), instance.getSettings());
+    }
+
+    public void testPerformAction() throws Exception {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        UpdateSettingsStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertThat(request.settings(), equalTo(step.getSettings()));
+                assertThat(request.indices(), equalTo(new String[] {indexMetaData.getIndex().getName()}));
+                listener.onResponse(new AcknowledgedResponse(true));
+                return null;
+            }
+
+        }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> actionCompleted = new SetOnce<>();
+
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                actionCompleted.set(complete);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("Unexpected method call", e);
+            }
+        });
+
+        assertEquals(true, actionCompleted.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any());
+    }
+
+    public void testPerformActionFailure() {
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        Exception exception = new RuntimeException();
+        UpdateSettingsStep step = createRandomInstance();
+
+        AdminClient adminClient = Mockito.mock(AdminClient.class);
+        IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
+
+        Mockito.when(client.admin()).thenReturn(adminClient);
+        Mockito.when(adminClient.indices()).thenReturn(indicesClient);
+        Mockito.doAnswer(new Answer<Void>() {
+
+            @Override
+            public Void answer(InvocationOnMock invocation) throws Throwable {
+                UpdateSettingsRequest request = (UpdateSettingsRequest) invocation.getArguments()[0];
+                @SuppressWarnings("unchecked")
+                ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
+                assertThat(request.settings(), equalTo(step.getSettings()));
+                assertThat(request.indices(), equalTo(new String[] {indexMetaData.getIndex().getName()}));
+                listener.onFailure(exception);
+                return null;
+            }
+
+        }).when(indicesClient).updateSettings(Mockito.any(), Mockito.any());
+
+        SetOnce<Boolean> exceptionThrown = new SetOnce<>();
+        step.performAction(indexMetaData, null, new Listener() {
+
+            @Override
+            public void onResponse(boolean complete) {
+                throw new AssertionError("Unexpected method call");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
+                exceptionThrown.set(true);
+            }
+        });
+
+        assertEquals(true, exceptionThrown.get());
+
+        Mockito.verify(client, Mockito.only()).admin();
+        Mockito.verify(adminClient, Mockito.only()).indices();
+        Mockito.verify(indicesClient, Mockito.only()).updateSettings(Mockito.any(), Mockito.any());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleRequestTests.java
new file mode 100644
index 0000000000000..c864bd76eeac5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleRequestTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Request;
+
+public class DeleteLifecycleRequestTests extends AbstractStreamableTestCase<DeleteLifecycleAction.Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(randomAlphaOfLengthBetween(1, 20));
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request mutateInstance(Request request) {
+        return new Request(request.getPolicyName() + randomAlphaOfLengthBetween(1, 10));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java
new file mode 100644
index 0000000000000..b671f72cf99c4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Response;
+
+public class DeleteLifecycleResponseTests extends AbstractStreamableTestCase<DeleteLifecycleAction.Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    @Override
+    protected Response mutateInstance(Response response) {
+        return new Response(response.isAcknowledged() == false);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleRequestTests.java
new file mode 100644
index 0000000000000..49caa0b48894e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleRequestTests.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Request;
+
+import java.util.Arrays;
+
+public class GetLifecycleRequestTests extends AbstractStreamableTestCase<GetLifecycleAction.Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(randomAlphaOfLengthBetween(1, 20));
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request mutateInstance(Request request) {
+        String[] originalPolicies = request.getPolicyNames();
+        String[] newPolicies = Arrays.copyOf(originalPolicies, originalPolicies.length + 1);
+        newPolicies[originalPolicies.length] = randomAlphaOfLength(5);
+        return new Request(newPolicies);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleResponseTests.java
new file mode 100644
index 0000000000000..08688407b3db6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleResponseTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.TestLifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.LifecyclePolicyResponseItem;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Response;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests.randomTestLifecyclePolicy;
+
+public class GetLifecycleResponseTests extends AbstractStreamableTestCase<GetLifecycleAction.Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        String randomPrefix = randomAlphaOfLength(5);
+        List<LifecyclePolicyResponseItem> responseItems = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(0, 2); i++) {
+            responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomPrefix + i),
+                randomNonNegativeLong(), randomAlphaOfLength(8)));
+        }
+        return new Response(responseItems);
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Arrays.asList(new NamedWriteableRegistry.Entry(LifecycleAction.class, MockAction.NAME, MockAction::new),
+                        new NamedWriteableRegistry.Entry(LifecycleType.class, TestLifecycleType.TYPE, in -> TestLifecycleType.INSTANCE)));
+    }
+
+    @Override
+    protected Response mutateInstance(Response response) {
+        List<LifecyclePolicyResponseItem> responseItems = new ArrayList<>(response.getPolicies());
+        if (responseItems.size() > 0) {
+            if (randomBoolean()) {
+                responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomAlphaOfLength(5)),
+                    randomNonNegativeLong(), randomAlphaOfLength(4)));
+            } else {
+                responseItems.remove(0);
+            }
+        } else {
+            responseItems.add(new LifecyclePolicyResponseItem(randomTestLifecyclePolicy(randomAlphaOfLength(2)),
+                randomNonNegativeLong(), randomAlphaOfLength(4)));
+        }
+        return new Response(responseItems);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepRequestTests.java
new file mode 100644
index 0000000000000..84b966b402323
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepRequestTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.StepKeyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Request;
+import org.junit.Before;
+
+public class MoveToStepRequestTests extends AbstractStreamableXContentTestCase<Request> {
+
+    private String index;
+    private static final StepKeyTests stepKeyTests = new StepKeyTests();
+
+    @Before
+    public void setup() {
+        index = randomAlphaOfLength(5);
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(index, stepKeyTests.createTestInstance(), stepKeyTests.createTestInstance());
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request doParseInstance(XContentParser parser) {
+        return Request.parseRequest(index, parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected Request mutateInstance(Request request) {
+        String index = request.getIndex();
+        StepKey currentStepKey = request.getCurrentStepKey();
+        StepKey nextStepKey = request.getNextStepKey();
+
+        switch (between(0, 2)) {
+            case 0:
+                index += randomAlphaOfLength(5);
+                break;
+            case 1:
+                currentStepKey = stepKeyTests.mutateInstance(currentStepKey);
+                break;
+            case 2:
+                nextStepKey = stepKeyTests.mutateInstance(nextStepKey);
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+
+        return new Request(index, currentStepKey, nextStepKey);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java
new file mode 100644
index 0000000000000..7f9bacd35ac25
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Response;
+
+public class MoveToStepResponseTests extends AbstractStreamableTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    @Override
+    protected Response mutateInstance(Response response) {
+        return new Response(response.isAcknowledged() == false);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java
new file mode 100644
index 0000000000000..5df60a7333143
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleRequestTests.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class PutLifecycleRequestTests extends AbstractStreamableXContentTestCase<PutLifecycleAction.Request> {
+
+    private String lifecycleName;
+
+    @Before
+    public void setup() {
+        lifecycleName = randomAlphaOfLength(20);
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(lifecycleName));
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request doParseInstance(XContentParser parser) {
+        return PutLifecycleAction.Request.parseRequest(lifecycleName, parser);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Arrays.asList(
+                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                    (in) -> TimeseriesLifecycleType.INSTANCE),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new)
+            ));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE),
+                (p) -> TimeseriesLifecycleType.INSTANCE),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected Request mutateInstance(Request request) {
+        String name = randomBoolean() ? lifecycleName : randomAlphaOfLength(5);
+        LifecyclePolicy policy = randomValueOtherThan(request.getPolicy(),
+            () -> LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(name));
+        return new Request(policy);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java
new file mode 100644
index 0000000000000..0c9acb2aa5dd2
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
+
+public class PutLifecycleResponseTests extends AbstractStreamableTestCase<PutLifecycleAction.Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    @Override
+    protected Response mutateInstance(Response response) {
+        return new Response(response.isAcknowledged() == false);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyRequestTests.java
new file mode 100644
index 0000000000000..e82f84564cae7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyRequestTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction.Request;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class RemoveIndexLifecyclePolicyRequestTests extends AbstractStreamableTestCase<RemoveIndexLifecyclePolicyAction.Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request(generateRandomStringArray(20, 20, false));
+        if (randomBoolean()) {
+            IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            request.indicesOptions(indicesOptions);
+        }
+        if (randomBoolean()) {
+            request.indices(generateRandomStringArray(20, 20, false));
+        }
+        return request;
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) throws IOException {
+        String[] indices = instance.indices();
+        IndicesOptions indicesOptions = instance.indicesOptions();
+        switch (between(0, 1)) {
+        case 0:
+            indices = randomValueOtherThanMany(i -> Arrays.equals(i, instance.indices()),
+                    () -> generateRandomStringArray(20, 20, false));
+            break;
+        case 1:
+            indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        Request newRequest = new Request(indices);
+        newRequest.indicesOptions(indicesOptions);
+        return newRequest;
+    }
+
+    public void testNullIndices() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> new Request((String[]) null));
+        assertEquals("indices cannot be null", exception.getMessage());
+    }
+
+    public void testValidate() {
+        Request request = createTestInstance();
+        assertNull(request.validate());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyResponseTests.java
new file mode 100644
index 0000000000000..a394e593e7307
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyResponseTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction.Response;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class RemoveIndexLifecyclePolicyResponseTests extends AbstractStreamableXContentTestCase<RemoveIndexLifecyclePolicyAction.Response> {
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    @Override
+    protected Response createTestInstance() {
+        List<String> failedIndexes = Arrays.asList(generateRandomStringArray(20, 20, false));
+        return new Response(failedIndexes);
+    }
+
+    @Override
+    protected Response mutateInstance(Response instance) throws IOException {
+        List<String> failedIndices = randomValueOtherThan(instance.getFailedIndexes(),
+                () -> Arrays.asList(generateRandomStringArray(20, 20, false)));
+        return new Response(failedIndices);
+    }
+
+    @Override
+    protected Response doParseInstance(XContentParser parser) throws IOException {
+        return Response.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testNullFailedIndices() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new Response((List<String>) null));
+        assertEquals("failed_indexes cannot be null", exception.getMessage());
+    }
+
+    public void testHasFailures() {
+        Response response = new Response(new ArrayList<>());
+        assertFalse(response.hasFailures());
+        assertEquals(Collections.emptyList(), response.getFailedIndexes());
+
+        int size = randomIntBetween(1, 10);
+        List<String> failedIndexes = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            failedIndexes.add(randomAlphaOfLength(20));
+        }
+        response = new Response(failedIndexes);
+        assertTrue(response.hasFailures());
+        assertEquals(failedIndexes, response.getFailedIndexes());
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryRequestTests.java
new file mode 100644
index 0000000000000..734bcf0b7df36
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryRequestTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Request;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class RetryRequestTests extends AbstractStreamableTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request();
+        if (randomBoolean()) {
+            request.indices(generateRandomStringArray(20, 20, false));
+        }
+        if (randomBoolean()) {
+            IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            request.indicesOptions(indicesOptions);
+        }
+        return request;
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) throws IOException {
+        String[] indices = instance.indices();
+        IndicesOptions indicesOptions = instance.indicesOptions();
+        switch (between(0, 1)) {
+            case 0:
+                indices = randomValueOtherThanMany(i -> Arrays.equals(i, instance.indices()),
+                    () -> generateRandomStringArray(20, 10, false, true));
+                break;
+            case 1:
+                indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(),
+                    randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+                break;
+            default:
+                throw new AssertionError("Illegal randomisation branch");
+        }
+        Request newRequest = new Request();
+        newRequest.indices(indices);
+        newRequest.indicesOptions(indicesOptions);
+        return newRequest;
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java
new file mode 100644
index 0000000000000..24e758f9503c1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+package org.elasticsearch.xpack.core.indexlifecycle.action;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Response;
+
+public class RetryResponseTests extends AbstractStreamableTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+
+    @Override
+    protected Response mutateInstance(Response response) {
+        return new Response(response.isAcknowledged() == false);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionResponseTests.java
index 77d4d788db620..7f4866982317a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionResponseTests.java
@@ -5,14 +5,16 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
+import org.elasticsearch.client.ml.PutCalendarResponse;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.core.ml.calendars.CalendarTests;
 
 import java.io.IOException;
 
-public class PutCalendarActionResponseTests extends AbstractStreamableXContentTestCase<PutCalendarAction.Response> {
+public class PutCalendarActionResponseTests
+    extends AbstractHlrcStreamableXContentTestCase<PutCalendarAction.Response, PutCalendarResponse> {
 
     @Override
     protected PutCalendarAction.Response createTestInstance() {
@@ -20,12 +22,29 @@ protected PutCalendarAction.Response createTestInstance() {
     }
 
     @Override
-    protected PutCalendarAction.Response createBlankInstance() {
-        return new PutCalendarAction.Response();
+    protected PutCalendarAction.Response doParseInstance(XContentParser parser) throws IOException {
+        return new PutCalendarAction.Response(Calendar.LENIENT_PARSER.parse(parser, null).build());
     }
 
     @Override
-    protected PutCalendarAction.Response doParseInstance(XContentParser parser) throws IOException {
-        return new PutCalendarAction.Response(Calendar.LENIENT_PARSER.parse(parser, null).build());
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    public PutCalendarResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+        return PutCalendarResponse.fromXContent(parser);
+    }
+
+    @Override
+    public PutCalendarAction.Response convertHlrcToInternal(PutCalendarResponse instance) {
+        org.elasticsearch.client.ml.calendars.Calendar hlrcCalendar = instance.getCalendar();
+        Calendar internalCalendar = new Calendar(hlrcCalendar.getId(), hlrcCalendar.getJobIds(), hlrcCalendar.getDescription());
+        return new PutCalendarAction.Response(internalCalendar);
+    }
+
+    @Override
+    protected PutCalendarAction.Response createBlankInstance() {
+        return new PutCalendarAction.Response();
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
index e71b0e5e8bdc1..340b399dc7f23 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
@@ -148,7 +148,7 @@ public void testPoints() throws Exception {
         assertEquals(Integer.BYTES, points.getBytesPerDimension());
 
         // number of dimensions
-        assertEquals(1, points.getNumDimensions());
+        assertEquals(1, points.getNumIndexDimensions());
 
         // walk the trees: we should see stuff in fieldA
         AtomicBoolean sawDoc = new AtomicBoolean(false);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index d8e0b693f7008..2290a34752819 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -307,7 +307,7 @@ public void testReloadingKeyStoreException() throws Exception {
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
-        new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService) {
+        new SSLConfigurationReloader(env, sslService, resourceWatcherService) {
             @Override
             void reloadSSLContext(SSLConfiguration configuration) {
                 fail("reload should not be called! [keystore reload exception]");
@@ -348,7 +348,7 @@ public void testReloadingPEMKeyConfigException() throws Exception {
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
-        new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService) {
+        new SSLConfigurationReloader(env, sslService, resourceWatcherService) {
             @Override
             void reloadSSLContext(SSLConfiguration configuration) {
                 fail("reload should not be called! [pem key reload exception]");
@@ -383,7 +383,7 @@ public void testTrustStoreReloadException() throws Exception {
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
-        new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService) {
+        new SSLConfigurationReloader(env, sslService, resourceWatcherService) {
             @Override
             void reloadSSLContext(SSLConfiguration configuration) {
                 fail("reload should not be called! [truststore reload exception]");
@@ -415,7 +415,7 @@ public void testPEMTrustReloadException() throws Exception {
         Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
-        new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService) {
+        new SSLConfigurationReloader(env, sslService, resourceWatcherService) {
             @Override
             void reloadSSLContext(SSLConfiguration configuration) {
                 fail("reload should not be called! [pem trust reload exception]");
@@ -444,7 +444,7 @@ private void validateSSLConfigurationIsReloaded(Settings settings, Environment e
         final CountDownLatch reloadLatch = new CountDownLatch(1);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.ssl");
-        new SSLConfigurationReloader(settings, env, sslService, resourceWatcherService) {
+        new SSLConfigurationReloader(env, sslService, resourceWatcherService) {
             @Override
             void reloadSSLContext(SSLConfiguration configuration) {
                 super.reloadSSLContext(configuration);
diff --git a/x-pack/plugin/graph/build.gradle b/x-pack/plugin/graph/build.gradle
index 069bfa5fbbe2b..f1bac2e54d455 100644
--- a/x-pack/plugin/graph/build.gradle
+++ b/x-pack/plugin/graph/build.gradle
@@ -15,6 +15,15 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+    project.subprojects
+            .find { it.path == project.path + ":qa" }
+            .subprojects
+            .findAll { it.path.startsWith(project.path + ":qa") }
+            .each { check.dependsOn it.check }
+}
+
 run {
     plugin xpackModule('core')
 }
diff --git a/x-pack/plugin/graph/qa/build.gradle b/x-pack/plugin/graph/qa/build.gradle
new file mode 100644
index 0000000000000..35bd236df5c23
--- /dev/null
+++ b/x-pack/plugin/graph/qa/build.gradle
@@ -0,0 +1,17 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+subprojects {
+    // HACK: please fix this
+    // we want to add the rest api specs for xpack to qa tests, but we
+    // need to wait until after the project is evaluated to only apply
+    // to those that rest tests. this used to be done automatically
+    // when xpack was a plugin, but now there is no place with xpack as a module.
+    // instead, we should package these and make them easy to use for rest tests,
+    // but currently, they must be copied into the resources of the test runner.
+    project.tasks.withType(RestIntegTestTask) {
+        File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')
+        project.copyRestSpec.from(xpackResources) {
+            include 'rest-api-spec/api/**'
+        }
+    }
+}
diff --git a/x-pack/qa/smoke-test-graph-with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle
similarity index 100%
rename from x-pack/qa/smoke-test-graph-with-security/build.gradle
rename to x-pack/plugin/graph/qa/with-security/build.gradle
diff --git a/x-pack/qa/smoke-test-graph-with-security/roles.yml b/x-pack/plugin/graph/qa/with-security/roles.yml
similarity index 100%
rename from x-pack/qa/smoke-test-graph-with-security/roles.yml
rename to x-pack/plugin/graph/qa/with-security/roles.yml
diff --git a/x-pack/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java b/x-pack/plugin/graph/qa/with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java
similarity index 100%
rename from x-pack/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java
rename to x-pack/plugin/graph/qa/with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java
diff --git a/x-pack/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityInsufficientRoleIT.java b/x-pack/plugin/graph/qa/with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityInsufficientRoleIT.java
similarity index 100%
rename from x-pack/qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityInsufficientRoleIT.java
rename to x-pack/plugin/graph/qa/with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityInsufficientRoleIT.java
diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle
new file mode 100644
index 0000000000000..71def8937817c
--- /dev/null
+++ b/x-pack/plugin/ilm/build.gradle
@@ -0,0 +1,34 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'x-pack-ilm'
+    description 'Elasticsearch Expanded Pack Plugin - Index Lifecycle Management'
+    classname 'org.elasticsearch.xpack.indexlifecycle.IndexLifecycle'
+    extendedPlugins = ['x-pack-core']
+    hasNativeController false
+    requiresKeystore true
+}
+archivesBaseName = 'x-pack-ilm'
+
+dependencies {
+    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+    project.subprojects
+            .find { it.path == project.path + ":qa" }
+            .subprojects
+            .findAll { it.path.startsWith(project.path + ":qa") }
+            .each { check.dependsOn it.check }
+}
+
+integTest.enabled = false
+
+run {
+    plugin xpackModule('core')
+}
diff --git a/x-pack/plugin/ilm/qa/build.gradle b/x-pack/plugin/ilm/qa/build.gradle
new file mode 100644
index 0000000000000..9525da5f4fed2
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/build.gradle
@@ -0,0 +1,20 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.build'
+test.enabled = false
+
+dependencies {
+    compile project(':test:framework')
+}
+
+subprojects {
+    project.tasks.withType(RestIntegTestTask) {
+        final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')
+        project.copyRestSpec.from(xPackResources) {
+            include 'rest-api-spec/api/**'
+        }
+    }
+}
+
+// the qa modules does not have any source files
+licenseHeaders.enabled = false
diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle
new file mode 100644
index 0000000000000..edd7f3aad472e
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle
@@ -0,0 +1,20 @@
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
+}
+
+integTestCluster {
+    numNodes = 4
+    clusterName = 'ilm'
+    
+    setting 'xpack.ilm.enabled', 'true'
+    setting 'xpack.security.enabled', 'false'
+    setting 'xpack.watcher.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'indices.lifecycle.poll_interval', '1000ms'
+
+}
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
new file mode 100644
index 0000000000000..0589da29312f8
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverStep;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+public class ChangePolicyforIndexIT extends ESRestTestCase {
+
+    /**
+     * This test aims to prove that an index will finish the current phase on an
+     * existing definition when the policy is changed for that index, and that
+     * after completing the current phase the new policy will be used for
+     * subsequent phases.
+     *
+     * The test creates two policies, one with a hot phase requiring 1 document
+     * to rollover and a warm phase with an impossible allocation action. The
+     * second policy has a rollover action requiring 1000 document and a warm
+     * phase that moves the index to known nodes that will succeed. An index is
+     * created with the fiorst policy set and the test ensures the policy is in
+     * the rollover step. It then changes the policy for the index to the second
+     * policy. It indexes a single document and checks that the index moves past
+     * the hot phase and through the warm phasee (proving the hot phase
+     * definition from the first policy was used) and then checks the allocation
+     * settings from the second policy are set ont he index (proving the second
+     * policy was used for the warm phase)
+     */
+    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35244")
+    public void testChangePolicyForIndex() throws Exception {
+        String indexName = "test-000001";
+        // create policy_1 and policy_2
+        Map<String, Phase> phases1 = new HashMap<>();
+        phases1.put("hot", new Phase("hot", TimeValue.ZERO, singletonMap(RolloverAction.NAME, new RolloverAction(null, null, 1L))));
+        phases1.put("warm", new Phase("warm", TimeValue.ZERO,
+                singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "foobarbaz"), null, null))));
+        LifecyclePolicy lifecyclePolicy1 = new LifecyclePolicy("policy_1", phases1);
+        Map<String, Phase> phases2 = new HashMap<>();
+        phases2.put("hot", new Phase("hot", TimeValue.ZERO, singletonMap(RolloverAction.NAME, new RolloverAction(null, null, 1000L))));
+        phases2.put("warm", new Phase("warm", TimeValue.ZERO,
+                singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null))));
+        LifecyclePolicy lifecyclePolicy2 = new LifecyclePolicy("policy_1", phases2);
+        // PUT policy_1 and policy_2
+        XContentBuilder builder1 = jsonBuilder();
+        lifecyclePolicy1.toXContent(builder1, null);
+        final StringEntity entity1 = new StringEntity("{ \"policy\":" + Strings.toString(builder1) + "}", ContentType.APPLICATION_JSON);
+        Request request1 = new Request("PUT", "_ilm/policy/" + "policy_1");
+        request1.setEntity(entity1);
+        assertOK(client().performRequest(request1));
+        XContentBuilder builder2 = jsonBuilder();
+        lifecyclePolicy2.toXContent(builder2, null);
+        final StringEntity entity2 = new StringEntity("{ \"policy\":" + Strings.toString(builder2) + "}", ContentType.APPLICATION_JSON);
+        Request request2 = new Request("PUT", "_ilm/policy/" + "policy_2");
+        request2.setEntity(entity2);
+        assertOK(client().performRequest(request2));
+
+        // create the test-index index and set the policy to policy_1
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.allocation.include._name", "node-0")
+                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias").put(LifecycleSettings.LIFECYCLE_NAME, "policy_1").build();
+        Request createIndexRequest = new Request("PUT", "/" + indexName);
+        createIndexRequest.setJsonEntity(
+                "{\n \"settings\": " + Strings.toString(settings) + ", \"aliases\" : { \"alias\": { \"is_write_index\": true } } }");
+        client().performRequest(createIndexRequest);
+        // wait for the shards to initialize
+        ensureGreen(indexName);
+
+        // Check the index is on the attempt rollover step
+        assertBusy(() -> assertStep(indexName, new StepKey("hot", RolloverAction.NAME, RolloverStep.NAME)));
+
+        // Change the policy to policy_2
+        Request changePolicyRequest = new Request("PUT", "/" + indexName + "/_settings");
+        final StringEntity changePolicyEntity = new StringEntity("{ \"index.lifecycle.name\": \"policy_2\" }",
+                ContentType.APPLICATION_JSON);
+        changePolicyRequest.setEntity(changePolicyEntity);
+        assertOK(client().performRequest(changePolicyRequest));
+
+        // Check the index is still on the attempt rollover step
+        assertBusy(() -> assertStep(indexName, new StepKey("hot", RolloverAction.NAME, RolloverStep.NAME)));
+
+        // Index a single document
+        XContentBuilder document = jsonBuilder().startObject();
+        document.field("foo", "bar");
+        document.endObject();
+        final Request request = new Request("POST", "/" + indexName + "/_doc/1");
+        request.setJsonEntity(Strings.toString(document));
+        assertOK(client().performRequest(request));
+
+        // Check the index goes to the warm phase and completes
+        assertBusy(() -> assertStep(indexName, TerminalPolicyStep.KEY));
+
+        // Check index is allocated on node-1 and node-2 as per policy_2
+        Request getSettingsRequest = new Request("GET", "/" + indexName + "/_settings");
+        Response getSettingsResponse = client().performRequest(getSettingsRequest);
+        assertOK(getSettingsResponse);
+        Map<String, Object> getSettingsResponseMap = entityAsMap(getSettingsResponse);
+        @SuppressWarnings("unchecked")
+        Map<String, Object> indexSettings = (Map<String, Object>) ((Map<String, Object>) getSettingsResponseMap.get(indexName))
+                .get("settings");
+        @SuppressWarnings("unchecked")
+        Map<String, Object> routingSettings = (Map<String, Object>) ((Map<String, Object>) indexSettings.get("index")).get("routing");
+        @SuppressWarnings("unchecked")
+        String includesAllocation = (String) ((Map<String, Object>) ((Map<String, Object>) routingSettings.get("allocation"))
+                .get("include")).get("_name");
+        assertEquals("node-1,node-2", includesAllocation);
+    }
+
+    public void testTempAwaitFix() {
+        // this is a test stub since there is only one test in this class and it is
+        // awaits-fixed. This test is to be removed once testChangePolicyForIndex is resolved
+    }
+
+    private void assertStep(String indexName, StepKey expectedStep) throws IOException {
+        Response explainResponse = client().performRequest(new Request("GET", "/" + indexName + "/_ilm/explain"));
+        assertOK(explainResponse);
+        Map<String, Object> explainResponseMap = entityAsMap(explainResponse);
+        @SuppressWarnings("unchecked")
+        Map<String, Object> indexExplainResponse = (Map<String, Object>) ((Map<String, Object>) explainResponseMap.get("indices"))
+                .get(indexName);
+        assertEquals(expectedStep.getPhase(), indexExplainResponse.get("phase"));
+        assertEquals(expectedStep.getAction(), indexExplainResponse.get("action"));
+        assertEquals(expectedStep.getName(), indexExplainResponse.get("step"));
+    }
+}
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
new file mode 100644
index 0000000000000..8dc8427bc7640
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
@@ -0,0 +1,518 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.not;
+
+public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
+    private String index;
+    private String policy;
+
+    @Before
+    public void refreshIndex() {
+        index = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        policy = randomAlphaOfLength(5);
+    }
+
+    public static void updatePolicy(String indexName, String policy) throws IOException {
+
+        Request changePolicyRequest = new Request("PUT", "/" + indexName + "/_settings");
+        final StringEntity changePolicyEntity = new StringEntity("{ \"index.lifecycle.name\": \"" + policy + "\" }",
+                ContentType.APPLICATION_JSON);
+        changePolicyRequest.setEntity(changePolicyEntity);
+        assertOK(client().performRequest(changePolicyRequest));
+    }
+
+    public void testFullPolicy() throws Exception {
+        String originalIndex = index + "-000001";
+        String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex;
+        String secondIndex = index + "-000002";
+        createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.routing.allocation.include._name", "node-0")
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
+
+        // create policy
+        createFullPolicy(TimeValue.ZERO);
+        // update policy on index
+        updatePolicy(originalIndex, policy);
+        // index document {"foo": "bar"} to trigger rollover
+        index(client(), originalIndex, "_id", "foo", "bar");
+
+        /*
+         * These asserts are in the order that they should be satisfied in, in
+         * order to maximize the time for all operations to complete.
+         * An "out of order" assert here may result in this test occasionally
+         * timing out and failing inappropriately.
+         */
+        // asserts that rollover was called
+        assertBusy(() -> assertTrue(indexExists(secondIndex)));
+        // asserts that shrink deleted the original index
+        assertBusy(() -> assertFalse(indexExists(originalIndex)));
+        // asserts that the delete phase completed for the managed shrunken index
+        assertBusy(() -> assertFalse(indexExists(shrunkenOriginalIndex)));
+    }
+
+    public void testMoveToAllocateStep() throws Exception {
+        String originalIndex = index + "-000001";
+        createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.routing.allocation.include._name", "node-0")
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
+
+        // create policy
+        createFullPolicy(TimeValue.timeValueHours(10));
+        // update policy on index
+        updatePolicy(originalIndex, policy);
+
+        // move to a step
+        Request moveToStepRequest = new Request("POST", "_ilm/move/" + originalIndex);
+        assertBusy(() -> assertTrue(getStepKeyForIndex(originalIndex).equals(new StepKey("new", "complete", "complete"))));
+        moveToStepRequest.setJsonEntity("{\n" +
+            "  \"current_step\": {\n" +
+            "    \"phase\": \"new\",\n" +
+            "    \"action\": \"complete\",\n" +
+            "    \"name\": \"complete\"\n" +
+            "  },\n" +
+            "  \"next_step\": {\n" +
+            "    \"phase\": \"cold\",\n" +
+            "    \"action\": \"allocate\",\n" +
+            "    \"name\": \"allocate\"\n" +
+            "  }\n" +
+            "}");
+        client().performRequest(moveToStepRequest);
+        assertBusy(() -> assertFalse(indexExists(originalIndex)));
+    }
+
+
+    public void testMoveToRolloverStep() throws Exception {
+        String originalIndex = index + "-000001";
+        String shrunkenOriginalIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + originalIndex;
+        String secondIndex = index + "-000002";
+        createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put("index.routing.allocation.include._name", "node-0")
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
+
+        createFullPolicy(TimeValue.timeValueHours(10));
+        // update policy on index
+        updatePolicy(originalIndex, policy);
+
+        // move to a step
+        Request moveToStepRequest = new Request("POST", "_ilm/move/" + originalIndex);
+        // index document to trigger rollover
+        index(client(), originalIndex, "_id", "foo", "bar");
+        logger.info(getStepKeyForIndex(originalIndex));
+        moveToStepRequest.setJsonEntity("{\n" +
+            "  \"current_step\": {\n" +
+            "    \"phase\": \"new\",\n" +
+            "    \"action\": \"complete\",\n" +
+            "    \"name\": \"complete\"\n" +
+            "  },\n" +
+            "  \"next_step\": {\n" +
+            "    \"phase\": \"hot\",\n" +
+            "    \"action\": \"rollover\",\n" +
+            "    \"name\": \"attempt_rollover\"\n" +
+            "  }\n" +
+            "}");
+        client().performRequest(moveToStepRequest);
+
+        /*
+         * These asserts are in the order that they should be satisfied in, in
+         * order to maximize the time for all operations to complete.
+         * An "out of order" assert here may result in this test occasionally
+         * timing out and failing inappropriately.
+         */
+        // asserts that rollover was called
+        assertBusy(() -> assertTrue(indexExists(secondIndex)));
+        // asserts that shrink deleted the original index
+        assertBusy(() -> assertFalse(indexExists(originalIndex)));
+        // asserts that the delete phase completed for the managed shrunken index
+        assertBusy(() -> assertFalse(indexExists(shrunkenOriginalIndex)));
+    }
+
+    public void testRolloverAction() throws Exception {
+        String originalIndex = index + "-000001";
+        String secondIndex = index + "-000002";
+        createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
+
+        // create policy
+        createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L));
+        // update policy on index
+        updatePolicy(originalIndex, policy);
+        // index document {"foo": "bar"} to trigger rollover
+        index(client(), originalIndex, "_id", "foo", "bar");
+        assertBusy(() -> assertTrue(indexExists(secondIndex)));
+        assertBusy(() -> assertTrue(indexExists(originalIndex)));
+    }
+
+    public void testRolloverAlreadyExists() throws Exception {
+        String originalIndex = index + "-000001";
+        String secondIndex = index + "-000002";
+        createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
+
+        // create policy
+        createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L));
+        // update policy on index
+        updatePolicy(originalIndex, policy);
+
+        // Manually create the new index
+        Request request = new Request("PUT", "/" + secondIndex);
+        request.setJsonEntity("{\n \"settings\": " + Strings.toString(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build()) + "}");
+        client().performRequest(request);
+        // wait for the shards to initialize
+        ensureGreen(secondIndex);
+
+        // index another doc to trigger the policy
+        index(client(), originalIndex, "_id", "foo", "bar");
+
+        assertBusy(() -> {
+            logger.info(originalIndex + ": " + getStepKeyForIndex(originalIndex));
+            logger.info(secondIndex + ": " + getStepKeyForIndex(secondIndex));
+            assertThat(getStepKeyForIndex(originalIndex), equalTo(new StepKey("hot", RolloverAction.NAME, ErrorStep.NAME)));
+            assertThat(getFailedStepForIndex(originalIndex), equalTo("update-rollover-lifecycle-date"));
+            assertThat(getReasonForIndex(originalIndex), equalTo("no rollover info found for [" + originalIndex + "], either the index " +
+                "has not yet rolled over or a subsequent index was created outside of Index Lifecycle Management"));
+        });
+    }
+
+    public void testAllocateOnlyAllocation() throws Exception {
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        String allocateNodeName = "node-" + randomFrom(0, 1);
+        AllocateAction allocateAction = new AllocateAction(null, null, null, singletonMap("_name", allocateNodeName));
+        createNewSingletonPolicy(randomFrom("warm", "cold"), allocateAction);
+        updatePolicy(index, policy);
+        assertBusy(() -> {
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
+        });
+        ensureGreen(index);
+    }
+
+    public void testAllocateActionOnlyReplicas() throws Exception {
+        int numShards = randomFrom(1, 5);
+        int numReplicas = randomFrom(0, 1);
+        int finalNumReplicas = (numReplicas + 1) % 2;
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas));
+        AllocateAction allocateAction = new AllocateAction(finalNumReplicas, null, null, null);
+        createNewSingletonPolicy(randomFrom("warm", "cold"), allocateAction);
+        updatePolicy(index, policy);
+        assertBusy(() -> {
+            Map<String, Object> settings = getOnlyIndexSettings(index);
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
+            assertThat(settings.get(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()), equalTo(String.valueOf(finalNumReplicas)));
+        });
+    }
+
+    public void testDelete() throws Exception {
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        createNewSingletonPolicy("delete", new DeleteAction());
+        updatePolicy(index, policy);
+        assertBusy(() -> assertFalse(indexExists(index)));
+    }
+
+    public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception {
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        createNewSingletonPolicy("delete", new DeleteAction(), TimeValue.timeValueHours(1));
+        updatePolicy(index, policy);
+        assertBusy(() -> {
+            assertThat(getStepKeyForIndex(index).getAction(), equalTo("complete"));
+            Map<String, Object> settings = getOnlyIndexSettings(index);
+            assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), not("true"));
+        });
+        indexDocument();
+    }
+
+    public void testReadOnly() throws Exception {
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        createNewSingletonPolicy("warm", new ReadOnlyAction());
+        updatePolicy(index, policy);
+        assertBusy(() -> {
+            Map<String, Object> settings = getOnlyIndexSettings(index);
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
+            assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testForceMergeAction() throws Exception {
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        for (int i = 0; i < randomIntBetween(2, 10); i++) {
+            Request request = new Request("PUT", index + "/_doc/" + i);
+            request.addParameter("refresh", "true");
+            request.setEntity(new StringEntity("{\"a\": \"test\"}", ContentType.APPLICATION_JSON));
+            client().performRequest(request);
+        }
+
+        Supplier<Integer> numSegments = () -> {
+            try {
+                Map<String, Object> segmentResponse = getAsMap(index + "/_segments");
+                segmentResponse = (Map<String, Object>) segmentResponse.get("indices");
+                segmentResponse = (Map<String, Object>) segmentResponse.get(index);
+                segmentResponse = (Map<String, Object>) segmentResponse.get("shards");
+                List<Map<String, Object>> shards = (List<Map<String, Object>>) segmentResponse.get("0");
+                return (Integer) shards.get(0).get("num_search_segments");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        };
+        assertThat(numSegments.get(), greaterThan(1));
+
+        createNewSingletonPolicy("warm", new ForceMergeAction(1));
+        updatePolicy(index, policy);
+
+        assertBusy(() -> {
+            assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
+            Map<String, Object> settings = getOnlyIndexSettings(index);
+            assertThat(numSegments.get(), equalTo(1));
+            assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
+        });
+        expectThrows(ResponseException.class, this::indexDocument);
+    }
+
+    public void testShrinkAction() throws Exception {
+        int numShards = 6;
+        int divisor = randomFrom(2, 3, 6);
+        int expectedFinalShards = numShards / divisor;
+        String shrunkenIndex = ShrinkAction.SHRUNKEN_INDEX_PREFIX + index;
+        createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numShards)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
+        createNewSingletonPolicy("warm", new ShrinkAction(expectedFinalShards));
+        updatePolicy(index, policy);
+        assertBusy(() -> {
+            assertTrue(indexExists(shrunkenIndex));
+            assertTrue(aliasExists(shrunkenIndex, index));
+            Map<String, Object> settings = getOnlyIndexSettings(shrunkenIndex);
+            assertThat(getStepKeyForIndex(shrunkenIndex), equalTo(TerminalPolicyStep.KEY));
+            assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(expectedFinalShards)));
+            assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
+        });
+        expectThrows(ResponseException.class, this::indexDocument);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testNonexistentPolicy() throws Exception {
+        String indexPrefix = randomAlphaOfLengthBetween(5,15).toLowerCase(Locale.ROOT);
+        final StringEntity template = new StringEntity("{\n" +
+            "  \"index_patterns\": \"" + indexPrefix + "*\",\n" +
+            "  \"settings\": {\n" +
+            "    \"index\": {\n" +
+            "      \"lifecycle\": {\n" +
+            "        \"name\": \"does_not_exist\",\n" +
+            "        \"rollover_alias\": \"test_alias\"\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", ContentType.APPLICATION_JSON);
+        Request templateRequest = new Request("PUT", "_template/test");
+        templateRequest.setEntity(template);
+        client().performRequest(templateRequest);
+
+        policy = randomAlphaOfLengthBetween(5,20);
+        createNewSingletonPolicy("hot", new RolloverAction(null, null, 1L));
+
+        index = indexPrefix + "-000001";
+        final StringEntity putIndex = new StringEntity("{\n" +
+            "  \"aliases\": {\n" +
+            "    \"test_alias\": {\n" +
+            "      \"is_write_index\": true\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", ContentType.APPLICATION_JSON);
+        Request putIndexRequest = new Request("PUT", index);
+        putIndexRequest.setEntity(putIndex);
+        client().performRequest(putIndexRequest);
+        indexDocument();
+
+        assertBusy(() -> {
+            Request explainRequest = new Request("GET", index + "/_ilm/explain");
+            Response response = client().performRequest(explainRequest);
+            Map<String, Object> responseMap;
+            try (InputStream is = response.getEntity().getContent()) {
+                responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+            }
+            logger.info(responseMap);
+            Map<String, Object> indexStatus = (Map<String, Object>)((Map<String, Object>) responseMap.get("indices")).get(index);
+            assertNull(indexStatus.get("phase"));
+            assertNull(indexStatus.get("action"));
+            assertNull(indexStatus.get("step"));
+            Map<String, String> stepInfo = (Map<String, String>) indexStatus.get("step_info");
+            assertNotNull(stepInfo);
+            assertEquals("policy [does_not_exist] does not exist", stepInfo.get("reason"));
+            assertEquals("illegal_argument_exception", stepInfo.get("type"));
+        });
+
+    }
+
+    private void createFullPolicy(TimeValue hotTime) throws IOException {
+        Map<String, LifecycleAction> warmActions = new HashMap<>();
+        warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1));
+        warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null));
+        warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
+        Map<String, Phase> phases = new HashMap<>();
+        phases.put("hot", new Phase("hot", hotTime, singletonMap(RolloverAction.NAME,
+            new RolloverAction(null, null, 1L))));
+        phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));
+        phases.put("cold", new Phase("cold", TimeValue.ZERO, singletonMap(AllocateAction.NAME,
+            new AllocateAction(0, singletonMap("_name", "node-3"), null, null))));
+        phases.put("delete", new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction())));
+        LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases);
+        // PUT policy
+        XContentBuilder builder = jsonBuilder();
+        lifecyclePolicy.toXContent(builder, null);
+        final StringEntity entity = new StringEntity(
+            "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON);
+        Request request = new Request("PUT", "_ilm/policy/" + policy);
+        request.setEntity(entity);
+        assertOK(client().performRequest(request));
+    }
+
+    private void createNewSingletonPolicy(String phaseName, LifecycleAction action) throws IOException {
+        createNewSingletonPolicy(phaseName, action, TimeValue.ZERO);
+    }
+
+    private void createNewSingletonPolicy(String phaseName, LifecycleAction action, TimeValue after) throws IOException {
+        Phase phase = new Phase(phaseName, after, singletonMap(action.getWriteableName(), action));
+        LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, singletonMap(phase.getName(), phase));
+        XContentBuilder builder = jsonBuilder();
+        lifecyclePolicy.toXContent(builder, null);
+        final StringEntity entity = new StringEntity(
+            "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON);
+        Request request = new Request("PUT", "_ilm/policy/" + policy);
+        request.setEntity(entity);
+        client().performRequest(request);
+    }
+
+    private void createIndexWithSettings(String index, Settings.Builder settings) throws IOException {
+        // create the test-index index
+        Request request = new Request("PUT", "/" + index);
+        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build())
+            + ", \"aliases\" : { \"alias\": { \"is_write_index\": true } } }");
+        client().performRequest(request);
+        // wait for the shards to initialize
+        ensureGreen(index);
+
+    }
+
+    private static void index(RestClient client, String index, String id, Object... fields) throws IOException {
+        XContentBuilder document = jsonBuilder().startObject();
+        for (int i = 0; i < fields.length; i += 2) {
+            document.field((String) fields[i], fields[i + 1]);
+        }
+        document.endObject();
+        final Request request = new Request("POST", "/" + index + "/_doc/" + id);
+        request.setJsonEntity(Strings.toString(document));
+        assertOK(client.performRequest(request));
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> getOnlyIndexSettings(String index) throws IOException {
+        Map<String, Object> response = (Map<String, Object>) getIndexSettings(index).get(index);
+        if (response == null) {
+            return Collections.emptyMap();
+        }
+        return (Map<String, Object>) response.get("settings");
+    }
+
+    private StepKey getStepKeyForIndex(String indexName) throws IOException {
+        Map<String, Object> indexResponse = explainIndex(indexName);
+        if (indexResponse == null) {
+            return new StepKey(null, null, null);
+        }
+
+        String phase = (String) indexResponse.get("phase");
+        String action = (String) indexResponse.get("action");
+        String step = (String) indexResponse.get("step");
+        return new StepKey(phase, action, step);
+    }
+
+    private String getFailedStepForIndex(String indexName) throws IOException {
+        Map<String, Object> indexResponse = explainIndex(indexName);
+        if (indexResponse == null) return null;
+
+        return (String) indexResponse.get("failed_step");
+    }
+
+    @SuppressWarnings("unchecked")
+    private String getReasonForIndex(String indexName) throws IOException {
+        Map<String, Object> indexResponse = explainIndex(indexName);
+        if (indexResponse == null) return null;
+
+        return ((Map<String, String>) indexResponse.get("step_info")).get("reason");
+    }
+
+    private Map<String, Object> explainIndex(String indexName) throws IOException {
+        Request explainRequest = new Request("GET", indexName + "/_ilm/explain");
+        Response response = client().performRequest(explainRequest);
+        Map<String, Object> responseMap;
+        try (InputStream is = response.getEntity().getContent()) {
+            responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+        }
+
+        @SuppressWarnings("unchecked") Map<String, Object> indexResponse = ((Map<String, Map<String, Object>>) responseMap.get("indices"))
+            .get(indexName);
+        return indexResponse;
+    }
+
+    private void indexDocument() throws IOException {
+        Request indexRequest = new Request("POST", index + "/_doc");
+        indexRequest.setEntity(new StringEntity("{\"a\": \"test\"}", ContentType.APPLICATION_JSON));
+        Response response = client().performRequest(indexRequest);
+        logger.info(response.getStatusLine());
+    }
+}
diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle
new file mode 100644
index 0000000000000..0f1e277e70d26
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/build.gradle
@@ -0,0 +1,45 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+dependencies {
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    testCompile project(path: xpackModule('ilm'), configuration: 'runtime')
+}
+
+task restTest(type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+}
+
+def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
+                          password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
+
+
+restTestRunner {
+    systemProperty 'tests.rest.cluster.username', clusterCredentials.username
+    systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+}
+
+restTestCluster {
+    distribution 'zip'
+    setting 'xpack.ilm.enabled', 'true'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setupCommand 'setup-admin-user',
+            'bin/elasticsearch-users', 'useradd', clusterCredentials.username, '-p', clusterCredentials.password, '-r', 'superuser'
+    waitCondition = { node, ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+                dest: tmpFile.toString(),
+                username: clusterCredentials.username,
+                password: clusterCredentials.password,
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
+}
+
+check.dependsOn restTest
+test.enabled = false
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRestIT.java b/x-pack/plugin/ilm/qa/rest/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRestIT.java
new file mode 100644
index 0000000000000..f784e2b940bfe
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRestIT.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs
+public class IndexLifecycleRestIT extends ESClientYamlSuiteTestCase {
+
+    private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username"));
+    private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password"));
+
+    public IndexLifecycleRestIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        return Settings.builder()
+                .put(super.restClientSettings())
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+}
+
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/10_basic.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/10_basic.yml
new file mode 100644
index 0000000000000..385430c1bf704
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/10_basic.yml
@@ -0,0 +1,218 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+
+---
+"Test Basic Policy CRUD":
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "10s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "delete": {
+                   "min_age": "30s",
+                   "actions": {
+                     "delete": {}
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+  - match: { my_timeseries_lifecycle.version: 1 }
+  - is_true: my_timeseries_lifecycle.modified_date
+  - match: { my_timeseries_lifecycle.policy.phases.warm.min_age: "10s" }
+  - match: { my_timeseries_lifecycle.policy.phases.delete.min_age: "30s" }
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+---
+"Test Policy Update":
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "10s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "delete": {
+                   "min_age": "30s",
+                   "actions": {
+                     "delete": {}
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+  - match: { my_timeseries_lifecycle.version: 1 }
+  - is_true: my_timeseries_lifecycle.modified_date
+  - match: { my_timeseries_lifecycle.policy.phases.warm.min_age: "10s" }
+  - match: { my_timeseries_lifecycle.policy.phases.delete.min_age: "30s" }
+
+
+  - do:
+      indices.create:
+        index: my_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index2
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+  
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "300s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "delete": {
+                   "min_age": "600s",
+                   "actions": {
+                     "delete": {}
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+  - match: { my_timeseries_lifecycle.version: 2 }
+  - is_true: my_timeseries_lifecycle.modified_date
+  - match: { my_timeseries_lifecycle.policy.phases.warm.min_age: "300s" }
+  - match: { my_timeseries_lifecycle.policy.phases.delete.min_age: "600s" }
+
+  - do:
+      indices.delete:
+        index: my_index
+  - do:
+      indices.delete:
+        index: my_index2
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+---
+"Test Undeletable Policy In Use":
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "10s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "delete": {
+                   "min_age": "30s",
+                   "actions": {
+                     "delete": {}
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
+  - match: { my_timeseries_lifecycle.policy.phases.warm.min_age: "10s" }
+  - match: { my_timeseries_lifecycle.policy.phases.delete.min_age: "30s" }
+
+  - do:
+      indices.create:
+        index: my_timeseries_index
+        body:
+          settings:
+            index.lifecycle.name: "my_timeseries_lifecycle"
+
+  - do:
+      catch: bad_request
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Cannot delete policy [my_timeseries_lifecycle]. It is being used by at least one index [my_timeseries_index]" }
+
+  - do:
+      ilm.remove_policy:
+        index: my_timeseries_index
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/20_move_to_step.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/20_move_to_step.yml
new file mode 100644
index 0000000000000..57223188d655b
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/20_move_to_step.yml
@@ -0,0 +1,179 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "1000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": { }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index_no_policy
+
+---
+teardown:
+
+  - do:
+      indices.delete:
+        index: my_index
+
+  - do:
+      indices.delete:
+        index: my_index_no_policy
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+---
+"Test Basic Move To Step":
+
+  - do:
+      ilm.move_to_step:
+        index: "my_index"
+        body:
+          current_step:
+            phase: "new"
+            action: "complete"
+            name: "complete"
+          next_step:
+            phase: "warm"
+            action: "forcemerge"
+            name: "forcemerge"
+
+  - do:
+        ilm.explain_lifecycle:
+          index: "my_index"
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.step: "forcemerge" }
+  - match: { indices.my_index.action: "forcemerge" }
+  - match: { indices.my_index.phase: "warm" }
+
+---
+"Test Invalid Move To Step With Incorrect Current Step":
+
+  - do:
+      catch: bad_request
+      ilm.move_to_step:
+        index: "my_index"
+        body:
+          current_step:
+            phase: "warm"
+            action: "forcemerge"
+            name: "forcemerge"
+          next_step:
+            phase: "warm"
+            action: "forcemerge"
+            name: "forcemerge"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "index [my_index] is not on current step [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}]" }
+
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "my_index"
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.step: "complete" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.phase: "new" }
+
+---
+"Test Invalid Move To Step With Invalid Next Step":
+
+  - do:
+      catch: bad_request
+      ilm.move_to_step:
+        index: "my_index"
+        body:
+          current_step:
+            phase: "new"
+            action: "complete"
+            name: "complete"
+          next_step:
+            phase: "invalid"
+            action: "invalid"
+            name: "invalid"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "step [{\"phase\":\"invalid\",\"action\":\"invalid\",\"name\":\"invalid\"}] for index [my_index] with policy [my_moveable_timeseries_lifecycle] does not exist" }
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "my_index"
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.step: "complete" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.phase: "new" }
+
+---
+"Test Invalid Move To Step With Invalid Policy":
+
+  - do:
+      catch: bad_request
+      ilm.move_to_step:
+        index: "my_index_no_policy"
+        body:
+          current_step:
+            phase: "hot"
+            action: "pre-pre-readonly"
+            name: "after"
+          next_step:
+            phase: "warm"
+            action: "forcemerge"
+            name: "forcemerge"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "index [my_index_no_policy] is not associated with an Index Lifecycle Policy" }
+
+---
+"Test Invalid Move To Step With Invalid Index":
+
+  - do:
+      catch: bad_request
+      ilm.move_to_step:
+        index: "does_not_exist"
+        body:
+          current_step:
+            phase: "hot"
+            action: "pre-pre-readonly"
+            name: "after"
+          next_step:
+            phase: "warm"
+            action: "forcemerge"
+            name: "forcemerge"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "index [does_not_exist] does not exist" }
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/30_retry.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/30_retry.yml
new file mode 100644
index 0000000000000..c6bdfb2a05e14
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/30_retry.yml
@@ -0,0 +1,101 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "1000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": { }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_lifecycle"
+
+---
+teardown:
+
+  - do:
+      indices.delete:
+        index: my_index
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_lifecycle"
+
+---
+"Test Invalid Retry With Non-errored Policy":
+
+  - do:
+      indices.create:
+        index: my_index
+        body:
+          settings:
+            index.lifecycle.name: "my_lifecycle"
+
+  - do:
+      catch: bad_request
+      ilm.retry:
+        index: "my_index"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "cannot retry an action for an index [my_index] that has not encountered an error when running a Lifecycle Policy" }
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "my_index"
+  - match: { indices.my_index.policy: "my_lifecycle" }
+  - match: { indices.my_index.step: "complete" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.phase: "new" }
+
+
+---
+"Test Invalid Retry With No Policy":
+
+  - do:
+      indices.create:
+        index: my_index
+
+  - do:
+      catch: bad_request
+      ilm.retry:
+        index: "my_index"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "cannot retry an action for an index [my_index] that has not encountered an error when running a Lifecycle Policy" }
+
+---
+"Test Invalid Re-run With Invalid Index":
+  - do:
+      indices.create:
+        index: my_index
+
+  - do:
+      catch: bad_request
+      ilm.retry:
+        index: "does_not_exist"
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "index [does_not_exist] does not exist" }
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/40_explain_lifecycle.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/40_explain_lifecycle.yml
new file mode 100644
index 0000000000000..8c8206d861152
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/40_explain_lifecycle.yml
@@ -0,0 +1,263 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "1000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": { }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index2
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: another_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: unmanaged_index
+        body:
+          settings: {}
+
+  - do:
+      indices.create:
+        index: my_index_no_policy
+
+---
+teardown:
+
+  - do:
+      indices.delete:
+        index: my_index
+  - do:
+      indices.delete:
+        index: my_index2
+  - do:
+      indices.delete:
+        index: another_index
+  - do:
+      indices.delete:
+        index: unmanaged_index
+
+  - do:
+      indices.delete:
+        index: my_index_no_policy
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+---
+"Test Basic Lifecycle Explain":
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "my_index"
+
+  - is_true: indices.my_index.managed
+  - match: { indices.my_index.index: "my_index" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.phase: "new" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.step: "complete" }
+  - is_true: indices.my_index.phase_time_millis
+  - is_false: indices.my_index.failed_step
+  - is_false: indices.my_index.step_info
+  - is_false: indices.my_index.phase_execution
+
+  - is_false: indices.my_index2
+  - is_false: indices.another_index
+  - is_false: indices.unmanaged_index
+
+---
+"Test Wildcard Index Lifecycle Explain":
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "my_*"
+
+  - is_true: indices.my_index.managed
+  - match: { indices.my_index.index: "my_index" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.phase: "new" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.step: "complete" }
+  - is_true: indices.my_index.phase_time_millis
+  - is_false: indices.my_index.failed_step
+  - is_false: indices.my_index.step_info
+  - is_false: indices.my_index.phase_execution
+
+  - is_true: indices.my_index2.managed
+  - match: { indices.my_index2.index: "my_index2" }
+  - match: { indices.my_index2.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index2.phase: "new" }
+  - match: { indices.my_index2.action: "complete" }
+  - match: { indices.my_index2.step: "complete" }
+  - is_true: indices.my_index2.phase_time_millis
+  - is_false: indices.my_index2.failed_step
+  - is_false: indices.my_index2.step_info
+  - is_false: indices.my_index2.phase_execution
+
+  - is_false: indices.another_index
+  - is_false: indices.unmanaged_index
+
+
+---
+"Test All Indexes Lifecycle Explain":
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "*"
+
+  - is_true: indices.my_index.managed
+  - match: { indices.my_index.index: "my_index" }
+  - match: { indices.my_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index.phase: "new" }
+  - match: { indices.my_index.action: "complete" }
+  - match: { indices.my_index.step: "complete" }
+  - is_true: indices.my_index.phase_time_millis
+  - is_false: indices.my_index.failed_step
+  - is_false: indices.my_index.step_info
+  - is_false: indices.my_index.phase_execution
+
+  - is_true: indices.my_index2.managed
+  - match: { indices.my_index2.index: "my_index2" }
+  - match: { indices.my_index2.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.my_index2.phase: "new" }
+  - match: { indices.my_index2.action: "complete" }
+  - match: { indices.my_index2.step: "complete" }
+  - is_true: indices.my_index2.phase_time_millis
+  - is_false: indices.my_index2.failed_step
+  - is_false: indices.my_index2.step_info
+  - is_false: indices.my_index2.phase_execution
+
+  - is_true: indices.another_index.managed
+  - match: { indices.another_index.index: "another_index" }
+  - match: { indices.another_index.policy: "my_moveable_timeseries_lifecycle" }
+  - match: { indices.another_index.phase: "new" }
+  - match: { indices.another_index.action: "complete" }
+  - match: { indices.another_index.step: "complete" }
+  - is_true: indices.another_index.phase_time_millis
+  - is_false: indices.another_index.failed_step
+  - is_false: indices.another_index.step_info
+  - is_false: indices.another_index.phase_execution
+
+  - match: { indices.unmanaged_index.index: "unmanaged_index" }
+  - is_false: indices.unmanaged_index.managed
+  - is_false: indices.unmanaged_index.policy
+  - is_false: indices.unmanaged_index.phase
+  - is_false: indices.unmanaged_index.action
+  - is_false: indices.unmanaged_index.step
+  - is_false: indices.another_index.failed_step
+  - is_false: indices.another_index.step_info
+
+---
+"Test Unmanaged Index Lifecycle Explain":
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "unmanaged_index"
+
+  - match: { indices.unmanaged_index.index: "unmanaged_index" }
+  - is_false: indices.unmanaged_index.managed
+  - is_false: indices.unmanaged_index.policy
+  - is_false: indices.unmanaged_index.phase
+  - is_false: indices.unmanaged_index.action
+  - is_false: indices.unmanaged_index.step
+  - is_false: indices.unmanaged_index.phase_execution
+  - is_false: indices.another_index.failed_step
+  - is_false: indices.another_index.step_info
+  - is_false: indices.my_index
+  - is_false: indices.my_index2
+  - is_false: indices.another_index
+
+---
+"Test new phase still has phase_time":
+
+  - do:
+      ilm.put_lifecycle:
+        policy: "mypolicy"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": {}
+                 },
+                 "warm": {
+                   "min_age": "2000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      indices.create:
+        index: foo
+        body:
+          settings:
+            index.lifecycle.name: "mypolicy"
+
+  - do:
+      ilm.explain_lifecycle:
+        index: "foo"
+
+  - is_true: indices.foo.managed
+  - match: { indices.foo.index: "foo" }
+  - match: { indices.foo.policy: "mypolicy" }
+  - match: { indices.foo.phase: "new" }
+  - match: { indices.foo.action: "complete" }
+  - match: { indices.foo.step: "complete" }
+  - is_true: indices.foo.phase_time_millis
+  - is_false: indices.foo.failed_step
+  - is_false: indices.foo.step_info
+  - is_false: indices.foo.phase_execution
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_operation_mode.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_operation_mode.yml
new file mode 100644
index 0000000000000..e8abc5b2c8d4e
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_operation_mode.yml
@@ -0,0 +1,63 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+
+---
+"Test Changing Operation Modes":
+  - do:
+      ilm.get_status: {}
+  - match: { operation_mode: "RUNNING" }
+
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "10s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "delete": {
+                   "min_age": "30s",
+                   "actions": {
+                     "delete": {}
+                   }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_status: {}
+  - match: { operation_mode: "RUNNING" }
+
+  - do:
+      ilm.stop: {}
+
+  - do:
+      ilm.get_status: {}
+  - match: { operation_mode: /STOPP(ED|ING)/ }
+
+  - do:
+      ilm.start: {}
+
+  - do:
+      ilm.get_status: {}
+  - match: { operation_mode: "RUNNING" }
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_timeseries_lifecycle"
diff --git a/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_remove_policy_for_index.yml b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_remove_policy_for_index.yml
new file mode 100644
index 0000000000000..c9537d9779733
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/60_remove_policy_for_index.yml
@@ -0,0 +1,203 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "1000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": { }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+        
+  - do:
+      ilm.put_lifecycle:
+        policy: "my_alternative_timeseries_lifecycle"
+        body: |
+           {
+             "policy": {
+               "phases": {
+                 "warm": {
+                   "min_age": "1000s",
+                   "actions": {
+                     "forcemerge": {
+                       "max_num_segments": 10000
+                     }
+                   }
+                 },
+                 "hot": {
+                   "min_age": "1000s",
+                   "actions": { }
+                 }
+               }
+             }
+           }
+
+  - do:
+      ilm.get_lifecycle:
+        policy: "my_alternative_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: my_index2
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: another_index
+        body:
+          settings:
+            index.lifecycle.name: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      indices.create:
+        index: unmanaged_index
+        body:
+          settings: {}
+
+  - do:
+      indices.create:
+        index: my_index_no_policy
+
+---
+teardown:
+
+  - do:
+      indices.delete:
+        index: my_index
+  - do:
+      indices.delete:
+        index: my_index2
+  - do:
+      indices.delete:
+        index: another_index
+  - do:
+      indices.delete:
+        index: unmanaged_index
+
+  - do:
+      indices.delete:
+        index: my_index_no_policy
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_moveable_timeseries_lifecycle"
+
+  - do:
+      ilm.delete_lifecycle:
+        policy: "my_alternative_timeseries_lifecycle"
+
+  - do:
+      catch: missing
+      ilm.get_lifecycle:
+        policy: "my_alternative_timeseries_lifecycle"
+
+---
+"Test Remove Policy Single Index":
+
+  - do:
+      indices.get_settings:
+        index: "another_index"
+  
+  - match: { another_index.settings.index.lifecycle.name: my_moveable_timeseries_lifecycle }
+
+  - do:
+      ilm.remove_policy:
+        index: "another_index"
+
+  - is_false: has_failures
+  - length: { failed_indexes: 0 }
+
+  - do:
+      indices.get_settings:
+        index: "another_index"
+
+  - is_false: another_index.settings.index.lifecycle
+  
+---
+"Test Remove Policy Index Pattern":
+
+  - do:
+      indices.get_settings:
+        index: "my_*"
+  
+  - match: { my_index.settings.index.lifecycle.name: my_moveable_timeseries_lifecycle }
+  - match: { my_index2.settings.index.lifecycle.name: my_moveable_timeseries_lifecycle }
+
+  - do:
+      ilm.remove_policy:
+        index: "my_*"
+
+  - is_false: has_failures
+  - length: { failed_indexes: 0 }
+
+  - do:
+      indices.get_settings:
+        index: "my_*"
+  
+  - is_false: my_index.settings.index.lifecycle
+  - is_false: my_index2.settings.index.lifecycle
+  
+---
+"Test Remove Policy Unmanaged Index":
+
+  - do:
+      indices.get_settings:
+        index: "unmanaged_index"
+  
+  - is_false: unmanaged_index.settings.index.lifecycle.name
+
+  - do:
+      ilm.remove_policy:
+        index: "unmanaged_index"
+
+  - is_false: has_failures
+  - length: { failed_indexes: 0 }
+
+  - do:
+      indices.get_settings:
+        index: "unmanaged_index"
+  
+  - is_false: unmanaged_index.settings.index.lifecycle
+  
+---
+"Test Remove Policy Index Does Not Exist":
+
+  - do:
+      catch: missing
+      ilm.remove_policy:
+        index: "doesnt_exist"
diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle
new file mode 100644
index 0000000000000..f1b972012e7f8
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/with-security/build.gradle
@@ -0,0 +1,43 @@
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
+}
+
+def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
+                          password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
+
+integTestRunner {
+  systemProperty 'tests.rest.cluster.username', clusterCredentials.username
+  systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+}
+
+integTestCluster {
+  setting 'xpack.ilm.enabled', 'true'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  extraConfigFile 'roles.yml', 'roles.yml'
+  setupCommand 'setupIlmUser',
+          'bin/elasticsearch-users',
+          'useradd', "test_ilm",
+          '-p', 'x-pack-test-password', '-r', "ilm"
+  setupCommand 'setupDummyUser',
+               'bin/elasticsearch-users',
+               'useradd', clusterCredentials.username,
+               '-p', clusterCredentials.password,
+               '-r', 'superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: clusterCredentials.username,
+            password: clusterCredentials.password,
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
+}
diff --git a/x-pack/plugin/ilm/qa/with-security/roles.yml b/x-pack/plugin/ilm/qa/with-security/roles.yml
new file mode 100644
index 0000000000000..baf89bea34568
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/with-security/roles.yml
@@ -0,0 +1,11 @@
+ilm:
+  cluster:
+    - monitor
+    - manage
+  indices:
+    - names: [ 'ilm-*' ]
+      privileges:
+        - monitor
+        - manage
+        - read
+        - write
\ No newline at end of file
diff --git a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
new file mode 100644
index 0000000000000..01eb07bb35b4f
--- /dev/null
+++ b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.equalTo;
+
+public class PermissionsIT extends ESRestTestCase {
+
+    private String deletePolicy = "deletePolicy";
+    private Settings indexSettingsWithPolicy;
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test_ilm", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Before
+    public void init() throws Exception {
+        Request request = new Request("PUT", "/_cluster/settings");
+        XContentBuilder pollIntervalEntity = JsonXContent.contentBuilder();
+        pollIntervalEntity.startObject();
+        pollIntervalEntity.startObject("transient");
+        pollIntervalEntity.field(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
+        pollIntervalEntity.endObject();
+        pollIntervalEntity.endObject();
+        request.setJsonEntity(Strings.toString(pollIntervalEntity));
+        assertOK(adminClient().performRequest(request));
+        indexSettingsWithPolicy = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_NAME, deletePolicy)
+            .put("number_of_shards", 1)
+            .put("number_of_replicas", 0)
+            .build();
+        createNewSingletonPolicy(deletePolicy,"delete", new DeleteAction());
+    }
+
+    /**
+     * Tests that a policy that simply deletes an index after 0s succeeds when an index
+     * with user `test_admin` is created referencing a policy created by `test_ilm` when both
+     * users have read/write permissions on the the index. The goal is to verify that one
+     * does not need to be the same user who created both the policy and the index to have the
+     * index be properly managed by ILM.
+     */
+    public void testCanManageIndexAndPolicyDifferentUsers() throws Exception {
+        String index = "ilm-00001";
+        createIndexAsAdmin(index, indexSettingsWithPolicy, "");
+        assertBusy(() -> assertFalse(indexExists(index)));
+    }
+
+    /**
+     * This tests the awkward behavior where an admin can have permissions to create a policy,
+     * but then not have permissions to operate on an index that was later associated with that policy by another
+     * user
+     */
+    @SuppressWarnings("unchecked")
+    public void testCanManageIndexWithNoPermissions() throws Exception {
+        createIndexAsAdmin("not-ilm", indexSettingsWithPolicy, "");
+        Request request = new Request("GET", "/not-ilm/_ilm/explain");
+        // test_ilm user does not have permissions on this index
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.FORBIDDEN.getStatus()));
+
+        assertBusy(() -> {
+            Response response = adminClient().performRequest(request);
+            assertOK(response);
+            try (InputStream is = response.getEntity().getContent()) {
+                Map<String, Object> mapResponse = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+                Map<String, Object> indexExplain = (Map<String, Object>) ((Map<String, Object>) mapResponse.get("indices")).get("not-ilm");
+                assertThat(indexExplain.get("managed"), equalTo(true));
+                assertThat(indexExplain.get("step"), equalTo("ERROR"));
+                assertThat(indexExplain.get("failed_step"), equalTo("delete"));
+                Map<String, String> stepInfo = (Map<String, String>) indexExplain.get("step_info");
+                assertThat(stepInfo.get("type"), equalTo("security_exception"));
+                assertThat(stepInfo.get("reason"), equalTo("action [indices:admin/delete] is unauthorized for user [test_ilm]"));
+            }
+        });
+    }
+
+    private void createNewSingletonPolicy(String policy, String phaseName, LifecycleAction action) throws IOException {
+        Phase phase = new Phase(phaseName, TimeValue.ZERO, singletonMap(action.getWriteableName(), action));
+        LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, singletonMap(phase.getName(), phase));
+        XContentBuilder builder = jsonBuilder();
+        lifecyclePolicy.toXContent(builder, null);
+        final StringEntity entity = new StringEntity(
+            "{ \"policy\":" + Strings.toString(builder) + "}", ContentType.APPLICATION_JSON);
+        Request request = new Request("PUT", "_ilm/policy/" + policy);
+        request.setEntity(entity);
+        client().performRequest(request);
+    }
+
+    private void createIndexAsAdmin(String name, Settings settings, String mapping) throws IOException {
+        Request request = new Request("PUT", "/" + name);
+        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
+            + ", \"mappings\" : {" + mapping + "} }");
+        assertOK(adminClient().performRequest(request));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java
new file mode 100644
index 0000000000000..9e5ef7b01c5f6
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTask.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+
+import java.io.IOException;
+import java.util.function.LongSupplier;
+
+public class ExecuteStepsUpdateTask extends ClusterStateUpdateTask {
+    private static final Logger logger = LogManager.getLogger(ExecuteStepsUpdateTask.class);
+    private final String policy;
+    private final Index index;
+    private final Step startStep;
+    private final PolicyStepsRegistry policyStepsRegistry;
+    private final IndexLifecycleRunner lifecycleRunner;
+    private LongSupplier nowSupplier;
+    private Step.StepKey nextStepKey = null;
+
+    public ExecuteStepsUpdateTask(String policy, Index index, Step startStep, PolicyStepsRegistry policyStepsRegistry,
+                                  IndexLifecycleRunner lifecycleRunner, LongSupplier nowSupplier) {
+        this.policy = policy;
+        this.index = index;
+        this.startStep = startStep;
+        this.policyStepsRegistry = policyStepsRegistry;
+        this.nowSupplier = nowSupplier;
+        this.lifecycleRunner = lifecycleRunner;
+    }
+
+    String getPolicy() {
+        return policy;
+    }
+
+    Index getIndex() {
+        return index;
+    }
+
+    Step getStartStep() {
+        return startStep;
+    }
+
+    Step.StepKey getNextStepKey() {
+        return nextStepKey;
+    }
+
+    /**
+     * {@link Step}s for the current index and policy are executed in succession until the next step to be
+     * executed is not a {@link ClusterStateActionStep}, or not a {@link ClusterStateWaitStep}, or does not
+     * belong to the same phase as the executed step. All other types of steps are executed outside of this
+     * {@link ClusterStateUpdateTask}, so they are of no concern here.
+     *
+     * @param currentState The current state to execute the <code>startStep</code> with
+     * @return the new cluster state after cluster-state operations and step transitions are applied
+     * @throws IOException if any exceptions occur
+     */
+    @Override
+    public ClusterState execute(final ClusterState currentState) throws IOException {
+        Step currentStep = startStep;
+        IndexMetaData indexMetaData = currentState.metaData().index(index);
+        if (indexMetaData == null) {
+            logger.debug("lifecycle for index [{}] executed but index no longer exists", index.getName());
+            // This index doesn't exist any more, there's nothing to execute currently
+            return currentState;
+        }
+        Step registeredCurrentStep = IndexLifecycleRunner.getCurrentStep(policyStepsRegistry, policy, indexMetaData,
+            LifecycleExecutionState.fromIndexMetadata(indexMetaData));
+        if (currentStep.equals(registeredCurrentStep)) {
+            ClusterState state = currentState;
+            // We can do cluster state steps all together until we
+            // either get to a step that isn't a cluster state step or a
+            // cluster state wait step returns not completed
+            while (currentStep instanceof ClusterStateActionStep || currentStep instanceof ClusterStateWaitStep) {
+                nextStepKey = currentStep.getNextStepKey();
+                if (currentStep instanceof ClusterStateActionStep) {
+                    // cluster state action step so do the action and
+                    // move the cluster state to the next step
+                    logger.trace("[{}] performing cluster state action ({}) [{}], next: [{}]",
+                        index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey(), currentStep.getNextStepKey());
+                    try {
+                        state = ((ClusterStateActionStep) currentStep).performAction(index, state);
+                    } catch (Exception exception) {
+                        return moveToErrorStep(state, currentStep.getKey(), exception);
+                    }
+                    if (currentStep.getNextStepKey() == null) {
+                        return state;
+                    } else {
+                        state = IndexLifecycleRunner.moveClusterStateToNextStep(index, state, currentStep.getKey(),
+                            currentStep.getNextStepKey(), nowSupplier);
+                    }
+                } else {
+                    // cluster state wait step so evaluate the
+                    // condition, if the condition is met move to the
+                    // next step, if its not met return the current
+                    // cluster state so it can be applied and we will
+                    // wait for the next trigger to evaluate the
+                    // condition again
+                    logger.trace("[{}] waiting for cluster state step condition ({}) [{}], next: [{}]",
+                        index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey(), currentStep.getNextStepKey());
+                    ClusterStateWaitStep.Result result;
+                    try {
+                        result = ((ClusterStateWaitStep) currentStep).isConditionMet(index, state);
+                    } catch (Exception exception) {
+                        return moveToErrorStep(state, currentStep.getKey(), exception);
+                    }
+                    if (result.isComplete()) {
+                        logger.trace("[{}] cluster state step condition met successfully ({}) [{}], moving to next step {}",
+                            index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey(), currentStep.getNextStepKey());
+                        if (currentStep.getNextStepKey() == null) {
+                            return state;
+                        } else {
+                            state = IndexLifecycleRunner.moveClusterStateToNextStep(index, state, currentStep.getKey(),
+                                currentStep.getNextStepKey(), nowSupplier);
+                        }
+                    } else {
+                        logger.trace("[{}] condition not met ({}) [{}], returning existing state",
+                            index.getName(), currentStep.getClass().getSimpleName(), currentStep.getKey());
+                        // We may have executed a step and set "nextStepKey" to
+                        // a value, but in this case, since the condition was
+                        // not met, we can't advance any way, so don't attempt
+                        // to run the current step
+                        nextStepKey = null;
+                        ToXContentObject stepInfo = result.getInfomationContext();
+                        if (stepInfo == null) {
+                            return state;
+                        } else {
+                            return IndexLifecycleRunner.addStepInfoToClusterState(index, state, stepInfo);
+                        }
+                    }
+                }
+                // There are actions we need to take in the event a phase
+                // transition happens, so even if we would continue in the while
+                // loop, if we are about to go into a new phase, return so that
+                // other processing can occur
+                if (currentStep.getKey().getPhase().equals(currentStep.getNextStepKey().getPhase()) == false) {
+                    return state;
+                }
+                currentStep = policyStepsRegistry.getStep(indexMetaData, currentStep.getNextStepKey());
+            }
+            return state;
+        } else {
+            // either we are no longer the master or the step is now
+            // not the same as when we submitted the update task. In
+            // either case we don't want to do anything now
+            return currentState;
+        }
+    }
+
+    @Override
+    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+        if (oldState.equals(newState) == false) {
+            IndexMetaData indexMetaData = newState.metaData().index(index);
+            if (nextStepKey != null && nextStepKey != TerminalPolicyStep.KEY && indexMetaData != null) {
+                logger.trace("[{}] step sequence starting with {} has completed, running next step {} if it is an async action",
+                    index.getName(), startStep.getKey(), nextStepKey);
+                // After the cluster state has been processed and we have moved
+                // to a new step, we need to conditionally execute the step iff
+                // it is an `AsyncAction` so that it is executed exactly once.
+                lifecycleRunner.maybeRunAsyncAction(newState, indexMetaData, policy, nextStepKey);
+            }
+        }
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        throw new ElasticsearchException(
+                "policy [" + policy + "] for index [" + index.getName() + "] failed on step [" + startStep.getKey() + "].", e);
+    }
+
+    private ClusterState moveToErrorStep(final ClusterState state, Step.StepKey currentStepKey, Exception cause) throws IOException {
+        logger.error("policy [{}] for index [{}] failed on cluster state step [{}]. Moving to ERROR step", policy, index.getName(),
+            currentStepKey);
+        MoveToErrorStepUpdateTask moveToErrorStepUpdateTask = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause,
+            nowSupplier);
+        return moveToErrorStepUpdateTask.execute(state);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java
new file mode 100644
index 0000000000000..1e42846b317d3
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java
@@ -0,0 +1,209 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestDeleteLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestExplainLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestGetLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestGetStatusAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestMoveToStepAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestPutLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestRemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestRetryAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestStartILMAction;
+import org.elasticsearch.xpack.indexlifecycle.action.RestStopAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportDeleteLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportExplainLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportGetLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportGetStatusAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportMoveToStepAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportPutLifecycleAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportRemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportRetryAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportStartILMAction;
+import org.elasticsearch.xpack.indexlifecycle.action.TransportStopILMAction;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.emptyList;
+
+public class IndexLifecycle extends Plugin implements ActionPlugin {
+    private final SetOnce<IndexLifecycleService> indexLifecycleInitialisationService = new SetOnce<>();
+    private Settings settings;
+    private boolean enabled;
+    private boolean transportClientMode;
+
+    public IndexLifecycle(Settings settings) {
+        this.settings = settings;
+        this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
+        this.transportClientMode = XPackPlugin.transportClientMode(settings);
+    }
+
+    // overridable by tests
+    protected Clock getClock() {
+        return Clock.systemUTC();
+    }
+
+    public Collection<Module> createGuiceModules() {
+        List<Module> modules = new ArrayList<>();
+
+        if (transportClientMode) {
+            return modules;
+        }
+
+        modules.add(b -> XPackPlugin.bindFeatureSet(b, IndexLifecycleFeatureSet.class));
+
+        return modules;
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+            LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
+            LifecycleSettings.LIFECYCLE_NAME_SETTING,
+            RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING);
+    }
+
+    @Override
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+        if (enabled == false || transportClientMode) {
+            return emptyList();
+        }
+        indexLifecycleInitialisationService
+            .set(new IndexLifecycleService(settings, client, clusterService, getClock(), System::currentTimeMillis, xContentRegistry));
+        return Collections.singletonList(indexLifecycleInitialisationService.get());
+    }
+
+    @Override
+    public List<Entry> getNamedWriteables() {
+        return Arrays.asList();
+    }
+
+    @Override
+    public List<org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry> getNamedXContent() {
+        return Arrays.asList(
+            // Custom Metadata
+            new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexLifecycleMetadata.TYPE),
+                parser -> IndexLifecycleMetadata.PARSER.parse(parser, null)),
+            // Lifecycle Types
+            new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE),
+                (p, c) -> TimeseriesLifecycleType.INSTANCE),
+            // Lifecycle Actions
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse)
+        );
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        if (enabled == false) {
+            return emptyList();
+        }
+        return Arrays.asList(
+                new RestPutLifecycleAction(settings, restController),
+                new RestGetLifecycleAction(settings, restController),
+                new RestDeleteLifecycleAction(settings, restController),
+                new RestExplainLifecycleAction(settings, restController),
+                new RestRemoveIndexLifecyclePolicyAction(settings, restController),
+                new RestMoveToStepAction(settings, restController),
+                new RestRetryAction(settings, restController),
+                new RestStopAction(settings, restController),
+                new RestStartILMAction(settings, restController),
+                new RestGetStatusAction(settings, restController)
+            );
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled == false) {
+            return emptyList();
+        }
+        return Arrays.asList(
+                new ActionHandler<>(PutLifecycleAction.INSTANCE, TransportPutLifecycleAction.class),
+                new ActionHandler<>(GetLifecycleAction.INSTANCE, TransportGetLifecycleAction.class),
+                new ActionHandler<>(DeleteLifecycleAction.INSTANCE, TransportDeleteLifecycleAction.class),
+                new ActionHandler<>(ExplainLifecycleAction.INSTANCE, TransportExplainLifecycleAction.class),
+                new ActionHandler<>(RemoveIndexLifecyclePolicyAction.INSTANCE, TransportRemoveIndexLifecyclePolicyAction.class),
+                new ActionHandler<>(MoveToStepAction.INSTANCE, TransportMoveToStepAction.class),
+                new ActionHandler<>(RetryAction.INSTANCE, TransportRetryAction.class),
+                new ActionHandler<>(StartILMAction.INSTANCE, TransportStartILMAction.class),
+                new ActionHandler<>(StopILMAction.INSTANCE, TransportStopILMAction.class),
+                new ActionHandler<>(GetStatusAction.INSTANCE, TransportGetStatusAction.class));
+    }
+
+    @Override
+    public void close() {
+        IndexLifecycleService lifecycleService = indexLifecycleInitialisationService.get();
+        if (lifecycleService != null) {
+            lifecycleService.close();
+        }
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java
new file mode 100644
index 0000000000000..2469621316889
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class IndexLifecycleFeatureSet implements XPackFeatureSet {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private ClusterService clusterService;
+
+    @Inject
+    public IndexLifecycleFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
+        this.clusterService = clusterService;
+        this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.INDEX_LIFECYCLE;
+    }
+
+    @Override
+    public String description() {
+        return "Index lifecycle management for the Elastic Stack";
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState != null && licenseState.isIndexLifecycleAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @Override
+    public Map<String, Object> nativeCodeInfo() {
+        return null;
+    }
+
+    @Override
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
+        MetaData metaData = clusterService.state().metaData();
+        IndexLifecycleMetadata lifecycleMetadata = metaData.custom(IndexLifecycleMetadata.TYPE);
+        if (enabled() && lifecycleMetadata != null) {
+            Map<String, Integer> policyUsage = new HashMap<>();
+            metaData.indices().forEach(entry -> {
+                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(entry.value.getSettings());
+                Integer indicesManaged = policyUsage.get(policyName);
+                if (indicesManaged == null) {
+                    indicesManaged = 1;
+                } else {
+                    indicesManaged = indicesManaged + 1;
+                }
+                policyUsage.put(policyName, indicesManaged);
+            });
+            List<PolicyStats> policyStats = lifecycleMetadata.getPolicies().values().stream().map(policy -> {
+                Map<String, PhaseStats> phaseStats = policy.getPhases().values().stream().map(phase -> {
+                    String[] actionNames = phase.getActions().keySet().toArray(new String[phase.getActions().size()]);
+                    return new Tuple<String, PhaseStats>(phase.getName(), new PhaseStats(phase.getMinimumAge(), actionNames));
+                }).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
+                return new PolicyStats(phaseStats, policyUsage.getOrDefault(policy.getName(), 0));
+            }).collect(Collectors.toList());
+            listener.onResponse(new IndexLifecycleFeatureSetUsage(available(), enabled(), policyStats));
+        } else {
+            listener.onResponse(new IndexLifecycleFeatureSetUsage(available(), enabled()));
+        }
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java
new file mode 100644
index 0000000000000..f5e33fdb98079
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunner.java
@@ -0,0 +1,510 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseCompleteStep;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.LongSupplier;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+
+public class IndexLifecycleRunner {
+    private static final Logger logger = LogManager.getLogger(IndexLifecycleRunner.class);
+    private PolicyStepsRegistry stepRegistry;
+    private ClusterService clusterService;
+    private LongSupplier nowSupplier;
+
+    public IndexLifecycleRunner(PolicyStepsRegistry stepRegistry, ClusterService clusterService, LongSupplier nowSupplier) {
+        this.stepRegistry = stepRegistry;
+        this.clusterService = clusterService;
+        this.nowSupplier = nowSupplier;
+    }
+
+    /**
+     * Return true or false depending on whether the index is ready to be in {@code phase}
+     */
+    boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetaData indexMetaData, final String phase) {
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        if (lifecycleState.getLifecycleDate() == null) {
+            logger.trace("no index creation date has been set yet");
+            return true;
+        }
+        final Long lifecycleDate = lifecycleState.getLifecycleDate();
+        assert lifecycleDate != null && lifecycleDate >= 0 : "expected index to have a lifecycle date but it did not";
+        final TimeValue after = stepRegistry.getIndexAgeForPhase(policy, phase);
+        final long now = nowSupplier.getAsLong();
+        final TimeValue age = new TimeValue(now - lifecycleDate);
+        if (logger.isTraceEnabled()) {
+            logger.trace("[{}] checking for index age to be at least [{}] before performing actions in " +
+                    "the \"{}\" phase. Now: {}, lifecycle date: {}, age: [{}/{}s]",
+                indexMetaData.getIndex().getName(), after, phase,
+                new TimeValue(now).seconds(),
+                new TimeValue(lifecycleDate).seconds(),
+                age, age.seconds());
+        }
+        return now >= lifecycleDate + after.getMillis();
+    }
+
+    /**
+     * Run the current step, only if it is an asynchronous wait step. These
+     * wait criteria are checked periodically from the ILM scheduler
+     */
+    public void runPeriodicStep(String policy, IndexMetaData indexMetaData) {
+        String index = indexMetaData.getIndex().getName();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        Step currentStep = getCurrentStep(stepRegistry, policy, indexMetaData, lifecycleState);
+        if (currentStep == null) {
+            if (stepRegistry.policyExists(policy) == false) {
+                markPolicyDoesNotExist(policy, indexMetaData.getIndex(), lifecycleState);
+                return;
+            } else {
+                logger.error("current step [{}] for index [{}] with policy [{}] is not recognized",
+                    getCurrentStepKey(lifecycleState), index, policy);
+                return;
+            }
+        }
+
+        if (currentStep instanceof TerminalPolicyStep) {
+            logger.debug("policy [{}] for index [{}] complete, skipping execution", policy, index);
+            return;
+        } else if (currentStep instanceof ErrorStep) {
+            logger.debug("policy [{}] for index [{}] on an error step, skipping execution", policy, index);
+            return;
+        }
+
+        logger.trace("[{}] maybe running periodic step ({}) with current step {}",
+            index, currentStep.getClass().getSimpleName(), currentStep.getKey());
+        // Only phase changing and async wait steps should be run through periodic polling
+        if (currentStep instanceof PhaseCompleteStep) {
+            // Only proceed to the next step if enough time has elapsed to go into the next phase
+            if (isReadyToTransitionToThisPhase(policy, indexMetaData, currentStep.getNextStepKey().getPhase())) {
+                moveToStep(indexMetaData.getIndex(), policy, currentStep.getKey(), currentStep.getNextStepKey());
+            }
+        } else if (currentStep instanceof AsyncWaitStep) {
+            logger.debug("[{}] running periodic policy with current-step [{}]", index, currentStep.getKey());
+            ((AsyncWaitStep) currentStep).evaluateCondition(indexMetaData, new AsyncWaitStep.Listener() {
+
+                @Override
+                public void onResponse(boolean conditionMet, ToXContentObject stepInfo) {
+                    logger.trace("cs-change-async-wait-callback, [{}] current-step: {}", index, currentStep.getKey());
+                    if (conditionMet) {
+                        moveToStep(indexMetaData.getIndex(), policy, currentStep.getKey(), currentStep.getNextStepKey());
+                    } else if (stepInfo != null) {
+                        setStepInfo(indexMetaData.getIndex(), policy, currentStep.getKey(), stepInfo);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    moveToErrorStep(indexMetaData.getIndex(), policy, currentStep.getKey(), e);
+                }
+            });
+        } else {
+            logger.trace("[{}] ignoring non periodic step execution from step transition [{}]", index, currentStep.getKey());
+        }
+    }
+
+    /**
+     * If the current step (matching the expected step key) is an asynchronous action step, run it
+     */
+    public void maybeRunAsyncAction(ClusterState currentState, IndexMetaData indexMetaData, String policy, StepKey expectedStepKey) {
+        String index = indexMetaData.getIndex().getName();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        Step currentStep = getCurrentStep(stepRegistry, policy, indexMetaData, lifecycleState);
+        if (currentStep == null) {
+            logger.warn("current step [{}] for index [{}] with policy [{}] is not recognized",
+                getCurrentStepKey(lifecycleState), index, policy);
+            return;
+        }
+
+        logger.trace("[{}] maybe running async action step ({}) with current step {}",
+            index, currentStep.getClass().getSimpleName(), currentStep.getKey());
+        if (currentStep.getKey().equals(expectedStepKey) == false) {
+            throw new IllegalStateException("expected index [" + indexMetaData.getIndex().getName() + "] with policy [" + policy +
+                "] to have current step consistent with provided step key (" + expectedStepKey + ") but it was " + currentStep.getKey());
+        }
+        if (currentStep instanceof AsyncActionStep) {
+            logger.debug("[{}] running policy with async action step [{}]", index, currentStep.getKey());
+            ((AsyncActionStep) currentStep).performAction(indexMetaData, currentState, new AsyncActionStep.Listener() {
+
+                @Override
+                public void onResponse(boolean complete) {
+                    logger.trace("cs-change-async-action-callback, [{}], current-step: {}", index, currentStep.getKey());
+                    if (complete && ((AsyncActionStep) currentStep).indexSurvives()) {
+                        moveToStep(indexMetaData.getIndex(), policy, currentStep.getKey(), currentStep.getNextStepKey());
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    moveToErrorStep(indexMetaData.getIndex(), policy, currentStep.getKey(), e);
+                }
+            });
+        } else {
+            logger.trace("[{}] ignoring non async action step execution from step transition [{}]", index, currentStep.getKey());
+        }
+    }
+
+    /**
+     * Run the current step that either waits for index age, or updates/waits-on cluster state.
+     * Invoked after the cluster state has been changed
+     */
+    public void runPolicyAfterStateChange(String policy, IndexMetaData indexMetaData) {
+        String index = indexMetaData.getIndex().getName();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        Step currentStep = getCurrentStep(stepRegistry, policy, indexMetaData, lifecycleState);
+        if (currentStep == null) {
+            if (stepRegistry.policyExists(policy) == false) {
+                markPolicyDoesNotExist(policy, indexMetaData.getIndex(), lifecycleState);
+                return;
+            } else {
+                logger.error("current step [{}] for index [{}] with policy [{}] is not recognized",
+                    getCurrentStepKey(lifecycleState), index, policy);
+                return;
+            }
+        }
+
+        if (currentStep instanceof TerminalPolicyStep) {
+            logger.debug("policy [{}] for index [{}] complete, skipping execution", policy, index);
+            return;
+        } else if (currentStep instanceof ErrorStep) {
+            logger.debug("policy [{}] for index [{}] on an error step, skipping execution", policy, index);
+            return;
+        }
+
+        logger.trace("[{}] maybe running step ({}) after state change: {}",
+            index, currentStep.getClass().getSimpleName(), currentStep.getKey());
+        if (currentStep instanceof PhaseCompleteStep) {
+            // Only proceed to the next step if enough time has elapsed to go into the next phase
+            if (isReadyToTransitionToThisPhase(policy, indexMetaData, currentStep.getNextStepKey().getPhase())) {
+                moveToStep(indexMetaData.getIndex(), policy, currentStep.getKey(), currentStep.getNextStepKey());
+            }
+        } else if (currentStep instanceof ClusterStateActionStep || currentStep instanceof ClusterStateWaitStep) {
+            logger.debug("[{}] running policy with current-step [{}]", indexMetaData.getIndex().getName(), currentStep.getKey());
+            clusterService.submitStateUpdateTask("ilm-execute-cluster-state-steps",
+                new ExecuteStepsUpdateTask(policy, indexMetaData.getIndex(), currentStep, stepRegistry, this, nowSupplier));
+        } else {
+            logger.trace("[{}] ignoring step execution from cluster state change event [{}]", index, currentStep.getKey());
+        }
+    }
+
+    /**
+     * Retrieves the current {@link StepKey} from the index settings. Note that
+     * it is illegal for the step to be set with the phase and/or action unset,
+     * or for the step to be unset with the phase and/or action set. All three
+     * settings must be either present or missing.
+     *
+     * @param lifecycleState the index custom data to extract the {@link StepKey} from.
+     */
+    public static StepKey getCurrentStepKey(LifecycleExecutionState lifecycleState) {
+        String currentPhase = lifecycleState.getPhase();
+        String currentAction = lifecycleState.getAction();
+        String currentStep = lifecycleState.getStep();
+        if (Strings.isNullOrEmpty(currentStep)) {
+            assert Strings.isNullOrEmpty(currentPhase) : "Current phase is not empty: " + currentPhase;
+            assert Strings.isNullOrEmpty(currentAction) : "Current action is not empty: " + currentAction;
+            return null;
+        } else {
+            assert Strings.isNullOrEmpty(currentPhase) == false;
+            assert Strings.isNullOrEmpty(currentAction) == false;
+            return new StepKey(currentPhase, currentAction, currentStep);
+        }
+    }
+
+    static Step getCurrentStep(PolicyStepsRegistry stepRegistry, String policy, IndexMetaData indexMetaData,
+                               LifecycleExecutionState lifecycleState) {
+        StepKey currentStepKey = getCurrentStepKey(lifecycleState);
+        logger.trace("[{}] retrieved current step key: {}", indexMetaData.getIndex().getName(), currentStepKey);
+        if (currentStepKey == null) {
+            return stepRegistry.getFirstStep(policy);
+        } else {
+            return stepRegistry.getStep(indexMetaData, currentStepKey);
+        }
+    }
+
+    /**
+     * This method is intended for handling moving to different steps from {@link TransportAction} executions.
+     * For this reason, it is reasonable to throw {@link IllegalArgumentException} when state is not as expected.
+     *
+     * @param indexName      The index whose step is to change
+     * @param currentState   The current {@link ClusterState}
+     * @param currentStepKey The current {@link StepKey} found for the index in the current cluster state
+     * @param nextStepKey    The next step to move the index into
+     * @param nowSupplier    The current-time supplier for updating when steps changed
+     * @param stepRegistry   The steps registry to check a step-key's existence in the index's current policy
+     * @return The updated cluster state where the index moved to <code>nextStepKey</code>
+     */
+    static ClusterState moveClusterStateToStep(String indexName, ClusterState currentState, StepKey currentStepKey,
+                                               StepKey nextStepKey, LongSupplier nowSupplier,
+                                               PolicyStepsRegistry stepRegistry) {
+        IndexMetaData idxMeta = currentState.getMetaData().index(indexName);
+        Settings indexSettings = idxMeta.getSettings();
+        String indexPolicySetting = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings);
+
+        // policy could be updated in-between execution
+        if (Strings.isNullOrEmpty(indexPolicySetting)) {
+            throw new IllegalArgumentException("index [" + indexName + "] is not associated with an Index Lifecycle Policy");
+        }
+
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta);
+        if (currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(lifecycleState)) == false) {
+            throw new IllegalArgumentException("index [" + indexName + "] is not on current step [" + currentStepKey + "]");
+        }
+
+        if (stepRegistry.stepExists(indexPolicySetting, nextStepKey) == false) {
+            throw new IllegalArgumentException("step [" + nextStepKey + "] for index [" + idxMeta.getIndex().getName() +
+                "] with policy [" + indexPolicySetting + "] does not exist");
+        }
+
+        return IndexLifecycleRunner.moveClusterStateToNextStep(idxMeta.getIndex(), currentState, currentStepKey, nextStepKey, nowSupplier);
+    }
+
+    static ClusterState moveClusterStateToNextStep(Index index, ClusterState clusterState, StepKey currentStep, StepKey nextStep,
+                                                   LongSupplier nowSupplier) {
+        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        LifecyclePolicyMetadata policyMetadata = ilmMeta.getPolicyMetadatas()
+            .get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()));
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMeta);
+        LifecycleExecutionState newLifecycleState = moveExecutionStateToNextStep(policyMetadata,
+            lifecycleState, currentStep, nextStep, nowSupplier);
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, newLifecycleState);
+
+        return newClusterStateBuilder.build();
+    }
+
+    static ClusterState moveClusterStateToErrorStep(Index index, ClusterState clusterState, StepKey currentStep, Exception cause,
+                                                    LongSupplier nowSupplier) throws IOException {
+        IndexMetaData idxMeta = clusterState.getMetaData().index(index);
+        IndexLifecycleMetadata ilmMeta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        LifecyclePolicyMetadata policyMetadata = ilmMeta.getPolicyMetadatas()
+            .get(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()));
+        XContentBuilder causeXContentBuilder = JsonXContent.contentBuilder();
+        causeXContentBuilder.startObject();
+        ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause);
+        causeXContentBuilder.endObject();
+        LifecycleExecutionState nextStepState = moveExecutionStateToNextStep(policyMetadata,
+            LifecycleExecutionState.fromIndexMetadata(idxMeta), currentStep, new StepKey(currentStep.getPhase(),
+                currentStep.getAction(), ErrorStep.NAME), nowSupplier);
+        LifecycleExecutionState.Builder failedState = LifecycleExecutionState.builder(nextStepState);
+        failedState.setFailedStep(currentStep.getName());
+        failedState.setStepInfo(BytesReference.bytes(causeXContentBuilder).utf8ToString());
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, failedState.build());
+        return newClusterStateBuilder.build();
+    }
+
+    ClusterState moveClusterStateToFailedStep(ClusterState currentState, String[] indices) {
+        ClusterState newState = currentState;
+        for (String index : indices) {
+            IndexMetaData indexMetaData = currentState.metaData().index(index);
+            if (indexMetaData == null) {
+                throw new IllegalArgumentException("index [" + index + "] does not exist");
+            }
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+            StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+            String failedStep = lifecycleState.getFailedStep();
+            if (currentStepKey != null && ErrorStep.NAME.equals(currentStepKey.getName())
+                && Strings.isNullOrEmpty(failedStep) == false) {
+                StepKey nextStepKey = new StepKey(currentStepKey.getPhase(), currentStepKey.getAction(), failedStep);
+                newState = moveClusterStateToStep(index, currentState, currentStepKey, nextStepKey, nowSupplier, stepRegistry);
+            } else {
+                throw new IllegalArgumentException("cannot retry an action for an index ["
+                    + index + "] that has not encountered an error when running a Lifecycle Policy");
+            }
+        }
+        return newState;
+    }
+
+    private static LifecycleExecutionState moveExecutionStateToNextStep(LifecyclePolicyMetadata policyMetadata,
+                                                                        LifecycleExecutionState existingState,
+                                                                        StepKey currentStep, StepKey nextStep,
+                                                                        LongSupplier nowSupplier) {
+        long nowAsMillis = nowSupplier.getAsLong();
+        LifecycleExecutionState.Builder updatedState = LifecycleExecutionState.builder(existingState);
+        updatedState.setPhase(nextStep.getPhase());
+        updatedState.setAction(nextStep.getAction());
+        updatedState.setStep(nextStep.getName());
+        updatedState.setStepTime(nowAsMillis);
+
+        // clear any step info or error-related settings from the current step
+        updatedState.setFailedStep(null);
+        updatedState.setStepInfo(null);
+
+        if (currentStep.getPhase().equals(nextStep.getPhase()) == false) {
+            final String newPhaseDefinition;
+            final Phase nextPhase;
+            if ("new".equals(nextStep.getPhase()) || TerminalPolicyStep.KEY.equals(nextStep)) {
+                nextPhase = null;
+            } else {
+                nextPhase = policyMetadata.getPolicy().getPhases().get(nextStep.getPhase());
+            }
+            PhaseExecutionInfo phaseExecutionInfo = new PhaseExecutionInfo(policyMetadata.getName(), nextPhase,
+                policyMetadata.getVersion(), policyMetadata.getModifiedDate());
+            newPhaseDefinition = Strings.toString(phaseExecutionInfo, false, false);
+            updatedState.setPhaseDefinition(newPhaseDefinition);
+            updatedState.setPhaseTime(nowAsMillis);
+        } else if (currentStep.getPhase().equals(InitializePolicyContextStep.INITIALIZATION_PHASE)) {
+            // The "new" phase is the initialization phase, usually the phase
+            // time would be set on phase transition, but since there is no
+            // transition into the "new" phase, we set it any time in the "new"
+            // phase
+            updatedState.setPhaseTime(nowAsMillis);
+        }
+
+        if (currentStep.getAction().equals(nextStep.getAction()) == false) {
+            updatedState.setActionTime(nowAsMillis);
+        }
+        return updatedState.build();
+    }
+
+    static ClusterState.Builder newClusterStateWithLifecycleState(Index index, ClusterState clusterState,
+                                                                  LifecycleExecutionState lifecycleState) {
+        ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
+        newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData())
+            .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap())));
+        return newClusterStateBuilder;
+    }
+
+    /**
+     * Conditionally updates cluster state with new step info. The new cluster state is only
+     * built if the step info has changed, otherwise the same old <code>clusterState</code> is
+     * returned
+     *
+     * @param index        the index to modify
+     * @param clusterState the cluster state to modify
+     * @param stepInfo     the new step info to update
+     * @return Updated cluster state with <code>stepInfo</code> if changed, otherwise the same cluster state
+     * if no changes to step info exist
+     * @throws IOException if parsing step info fails
+     */
+    static ClusterState addStepInfoToClusterState(Index index, ClusterState clusterState, ToXContentObject stepInfo) throws IOException {
+        IndexMetaData indexMetaData = clusterState.getMetaData().index(index);
+        if (indexMetaData == null) {
+            // This index doesn't exist anymore, we can't do anything
+            return clusterState;
+        }
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
+        final String stepInfoString;
+        try (XContentBuilder infoXContentBuilder = JsonXContent.contentBuilder()) {
+            stepInfo.toXContent(infoXContentBuilder, ToXContent.EMPTY_PARAMS);
+            stepInfoString = BytesReference.bytes(infoXContentBuilder).utf8ToString();
+        }
+        if (stepInfoString.equals(lifecycleState.getStepInfo())) {
+            return clusterState;
+        }
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder(lifecycleState);
+        newState.setStepInfo(stepInfoString);
+        ClusterState.Builder newClusterStateBuilder = newClusterStateWithLifecycleState(index, clusterState, newState.build());
+        return newClusterStateBuilder.build();
+    }
+
+    private void moveToStep(Index index, String policy, StepKey currentStepKey, StepKey nextStepKey) {
+        logger.debug("[{}] moving to step [{}] {} -> {}", index.getName(), policy, currentStepKey, nextStepKey);
+        clusterService.submitStateUpdateTask("ilm-move-to-step",
+            new MoveToNextStepUpdateTask(index, policy, currentStepKey, nextStepKey, nowSupplier, clusterState ->
+            {
+                IndexMetaData indexMetaData = clusterState.metaData().index(index);
+                if (nextStepKey != null && nextStepKey != TerminalPolicyStep.KEY && indexMetaData != null) {
+                    maybeRunAsyncAction(clusterState, indexMetaData, policy, nextStepKey);
+                }
+            }));
+    }
+
+    private void moveToErrorStep(Index index, String policy, StepKey currentStepKey, Exception e) {
+        logger.error(new ParameterizedMessage("policy [{}] for index [{}] failed on step [{}]. Moving to ERROR step",
+                policy, index.getName(), currentStepKey), e);
+        clusterService.submitStateUpdateTask("ilm-move-to-error-step",
+            new MoveToErrorStepUpdateTask(index, policy, currentStepKey, e, nowSupplier));
+    }
+
+    private void setStepInfo(Index index, String policy, StepKey currentStepKey, ToXContentObject stepInfo) {
+        clusterService.submitStateUpdateTask("ilm-set-step-info", new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo));
+    }
+
+    public static ClusterState removePolicyForIndexes(final Index[] indices, ClusterState currentState, List<String> failedIndexes) {
+        MetaData.Builder newMetadata = MetaData.builder(currentState.getMetaData());
+        boolean clusterStateChanged = false;
+        for (Index index : indices) {
+            IndexMetaData indexMetadata = currentState.getMetaData().index(index);
+            if (indexMetadata == null) {
+                // Index doesn't exist so fail it
+                failedIndexes.add(index.getName());
+            } else {
+                IndexMetaData.Builder newIdxMetadata = IndexLifecycleRunner.removePolicyForIndex(indexMetadata);
+                if (newIdxMetadata != null) {
+                    newMetadata.put(newIdxMetadata);
+                    clusterStateChanged = true;
+                }
+            }
+        }
+        if (clusterStateChanged) {
+            ClusterState.Builder newClusterState = ClusterState.builder(currentState);
+            newClusterState.metaData(newMetadata);
+            return newClusterState.build();
+        } else {
+            return currentState;
+        }
+    }
+
+    private static IndexMetaData.Builder removePolicyForIndex(IndexMetaData indexMetadata) {
+        Settings idxSettings = indexMetadata.getSettings();
+        Settings.Builder newSettings = Settings.builder().put(idxSettings);
+        boolean notChanged = true;
+
+        notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey()));
+        notChanged &= Strings.isNullOrEmpty(newSettings.remove(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.getKey()));
+        long newSettingsVersion = notChanged ? indexMetadata.getSettingsVersion() : 1 + indexMetadata.getSettingsVersion();
+
+        IndexMetaData.Builder builder = IndexMetaData.builder(indexMetadata);
+        builder.removeCustom(ILM_CUSTOM_METADATA_KEY);
+        return builder.settings(newSettings).settingsVersion(newSettingsVersion);
+    }
+
+    private void markPolicyDoesNotExist(String policyName, Index index, LifecycleExecutionState executionState) {
+        logger.debug("policy [{}] for index [{}] does not exist, recording this in step_info for this index",
+            policyName, index.getName());
+        setStepInfo(index, policyName, getCurrentStepKey(executionState),
+            new SetStepInfoUpdateTask.ExceptionWrapper(
+                new IllegalArgumentException("policy [" + policyName + "] does not exist")));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java
new file mode 100644
index 0000000000000..e20ca604de061
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java
@@ -0,0 +1,266 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.LocalNodeMasterListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
+
+import java.io.Closeable;
+import java.time.Clock;
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.LongSupplier;
+
+/**
+ * A service which runs the {@link LifecyclePolicy}s associated with indexes.
+ */
+public class IndexLifecycleService extends AbstractComponent
+        implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener {
+    private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class);
+    private static final Set<String> IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME);
+    private volatile boolean isMaster = false;
+    private volatile TimeValue pollInterval;
+
+    private final SetOnce<SchedulerEngine> scheduler = new SetOnce<>();
+    private final Clock clock;
+    private final PolicyStepsRegistry policyRegistry;
+    private final IndexLifecycleRunner lifecycleRunner;
+    private final Settings settings;
+    private Client client;
+    private ClusterService clusterService;
+    private LongSupplier nowSupplier;
+    private SchedulerEngine.Job scheduledJob;
+
+    public IndexLifecycleService(Settings settings, Client client, ClusterService clusterService, Clock clock, LongSupplier nowSupplier,
+                                 NamedXContentRegistry xContentRegistry) {
+        this.settings = settings;
+        this.client = client;
+        this.clusterService = clusterService;
+        this.clock = clock;
+        this.nowSupplier = nowSupplier;
+        this.scheduledJob = null;
+        this.policyRegistry = new PolicyStepsRegistry(xContentRegistry, client);
+        this.lifecycleRunner = new IndexLifecycleRunner(policyRegistry, clusterService, nowSupplier);
+        this.pollInterval = LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING.get(settings);
+        clusterService.addStateApplier(this);
+        clusterService.addListener(this);
+        clusterService.addLocalNodeMasterListener(this);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING,
+            this::updatePollInterval);
+    }
+
+    public void maybeRunAsyncAction(ClusterState clusterState, IndexMetaData indexMetaData, StepKey nextStepKey) {
+        String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMetaData.getSettings());
+        lifecycleRunner.maybeRunAsyncAction(clusterState, indexMetaData, policyName, nextStepKey);
+    }
+
+    public ClusterState moveClusterStateToStep(ClusterState currentState, String indexName, StepKey currentStepKey, StepKey nextStepKey) {
+        return IndexLifecycleRunner.moveClusterStateToStep(indexName, currentState, currentStepKey, nextStepKey,
+            nowSupplier, policyRegistry);
+    }
+
+    public ClusterState moveClusterStateToFailedStep(ClusterState currentState, String[] indices) {
+        return lifecycleRunner.moveClusterStateToFailedStep(currentState, indices);
+    }
+
+    @Override
+    public void onMaster() {
+        this.isMaster = true;
+        maybeScheduleJob();
+
+        ClusterState clusterState = clusterService.state();
+        IndexLifecycleMetadata currentMetadata = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        if (currentMetadata != null) {
+            OperationMode currentMode = currentMetadata.getOperationMode();
+            if (OperationMode.STOPPED.equals(currentMode)) {
+                return;
+            }
+
+            boolean safeToStop = true; // true until proven false by a run policy
+
+            // If we just became master, we need to kick off any async actions that
+            // may have not been run due to master rollover
+            for (ObjectCursor<IndexMetaData> cursor : clusterState.metaData().indices().values()) {
+                IndexMetaData idxMeta = cursor.value;
+                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
+                if (Strings.isNullOrEmpty(policyName) == false) {
+                    StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta));
+                    if (OperationMode.STOPPING == currentMode &&
+                        stepKey != null &&
+                        IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) {
+                        logger.info("skipping policy [{}] for index [{}]. stopping Index Lifecycle execution",
+                            policyName, idxMeta.getIndex().getName());
+                        continue;
+                    }
+                    lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey);
+                    safeToStop = false; // proven false!
+                }
+            }
+            if (safeToStop && OperationMode.STOPPING == currentMode) {
+                submitOperationModeUpdate(OperationMode.STOPPED);
+            }
+        }
+    }
+
+    @Override
+    public void offMaster() {
+        this.isMaster = false;
+        cancelJob();
+    }
+
+    @Override
+    public String executorName() {
+        return ThreadPool.Names.MANAGEMENT;
+    }
+
+    private void updatePollInterval(TimeValue newInterval) {
+        this.pollInterval = newInterval;
+        maybeScheduleJob();
+    }
+
+    // pkg-private for testing
+    SchedulerEngine getScheduler() {
+        return scheduler.get();
+    }
+
+    // pkg-private for testing
+    SchedulerEngine.Job getScheduledJob() {
+        return scheduledJob;
+    }
+
+    private void maybeScheduleJob() {
+        if (this.isMaster) {
+            if (scheduler.get() == null) {
+                scheduler.set(new SchedulerEngine(settings, clock));
+                scheduler.get().register(this);
+            }
+            scheduledJob = new SchedulerEngine.Job(XPackField.INDEX_LIFECYCLE, new TimeValueSchedule(pollInterval));
+            scheduler.get().add(scheduledJob);
+        }
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        IndexLifecycleMetadata lifecycleMetadata = event.state().metaData().custom(IndexLifecycleMetadata.TYPE);
+        if (this.isMaster && lifecycleMetadata != null) {
+            triggerPolicies(event.state(), true);
+        }
+    }
+
+    @Override
+    public void applyClusterState(ClusterChangedEvent event) {
+        if (event.localNodeMaster()) { // only act if we are master, otherwise
+                                       // keep idle until elected
+            if (event.state().metaData().custom(IndexLifecycleMetadata.TYPE) != null) {
+                policyRegistry.update(event.state());
+            }
+        }
+    }
+
+    private void cancelJob() {
+        if (scheduler.get() != null) {
+            scheduler.get().remove(XPackField.INDEX_LIFECYCLE);
+            scheduledJob = null;
+        }
+    }
+
+    @Override
+    public void triggered(SchedulerEngine.Event event) {
+        if (event.getJobName().equals(XPackField.INDEX_LIFECYCLE)) {
+            logger.trace("job triggered: " + event.getJobName() + ", " + event.getScheduledTime() + ", " + event.getTriggeredTime());
+            triggerPolicies(clusterService.state(), false);
+        }
+    }
+
+    /**
+     * executes the policy execution on the appropriate indices by running cluster-state tasks per index.
+     *
+     * If stopping ILM was requested, and it is safe to stop, this will also be done here
+     * when possible after no policies are executed.
+     *
+     * @param clusterState the current cluster state
+     * @param fromClusterStateChange whether things are triggered from the cluster-state-listener or the scheduler
+     */
+    void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
+        IndexLifecycleMetadata currentMetadata = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+
+        if (currentMetadata == null) {
+            return;
+        }
+
+        OperationMode currentMode = currentMetadata.getOperationMode();
+
+        if (OperationMode.STOPPED.equals(currentMode)) {
+            return;
+        }
+
+        boolean safeToStop = true; // true until proven false by a run policy
+
+        // loop through all indices in cluster state and filter for ones that are
+        // managed by the Index Lifecycle Service they have a index.lifecycle.name setting
+        // associated to a policy
+        for (ObjectCursor<IndexMetaData> cursor : clusterState.metaData().indices().values()) {
+            IndexMetaData idxMeta = cursor.value;
+            String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
+            if (Strings.isNullOrEmpty(policyName) == false) {
+                StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(LifecycleExecutionState.fromIndexMetadata(idxMeta));
+                if (OperationMode.STOPPING == currentMode && stepKey != null
+                        && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction()) == false) {
+                    logger.info("skipping policy [" + policyName + "] for index [" + idxMeta.getIndex().getName()
+                        + "]. stopping Index Lifecycle execution");
+                    continue;
+                }
+                if (fromClusterStateChange) {
+                    lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta);
+                } else {
+                    lifecycleRunner.runPeriodicStep(policyName, idxMeta);
+                }
+                safeToStop = false; // proven false!
+            }
+        }
+        if (safeToStop && OperationMode.STOPPING == currentMode) {
+            submitOperationModeUpdate(OperationMode.STOPPED);
+        }
+    }
+
+    @Override
+    public void close() {
+        SchedulerEngine engine = scheduler.get();
+        if (engine != null) {
+            engine.stop();
+        }
+    }
+
+    public void submitOperationModeUpdate(OperationMode mode) {
+        clusterService.submitStateUpdateTask("ilm_operation_mode_update",
+            new OperationModeUpdateTask(mode));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java
new file mode 100644
index 0000000000000..01ff0243bc93b
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.support.AbstractClient;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+
+import java.util.Map;
+
+/**
+ * This class wraps a client and calls the client using the headers provided in
+ * constructor. The intent is to abstract away the fact that there are headers
+ * so {@link Step}s etc. can call this client as if it was a normal client.
+ *
+ * Note: This client will not close the wrapped {@link Client} instance since
+ * the intent is that the wrapped client is shared between multiple instances of
+ * this class.
+ */
+public class LifecyclePolicySecurityClient extends AbstractClient {
+
+    private Client client;
+    private Map<String, String> headers;
+    private String origin;
+
+    public LifecyclePolicySecurityClient(Client client, String origin, Map<String, String> headers) {
+        super(client.settings(), client.threadPool());
+        this.client = client;
+        this.origin = origin;
+        this.headers = headers;
+    }
+
+    @Override
+    public void close() {
+        // Doesn't close the wrapped client since this client object is shared
+        // among multiple instances
+    }
+
+    @Override
+    protected <
+            Request extends ActionRequest,
+            Response extends ActionResponse,
+            RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>>
+    void doExecute(final Action<Request, Response, RequestBuilder> action, final Request request, final ActionListener<Response> listener) {
+        ClientHelper.executeWithHeadersAsync(headers, origin, client, action, request, listener);
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTask.java
new file mode 100644
index 0000000000000..5af1a05309e7e
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTask.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+
+import java.io.IOException;
+import java.util.function.LongSupplier;
+
+public class MoveToErrorStepUpdateTask extends ClusterStateUpdateTask {
+    private final Index index;
+    private final String policy;
+    private final Step.StepKey currentStepKey;
+    private LongSupplier nowSupplier;
+    private Exception cause;
+
+    public MoveToErrorStepUpdateTask(Index index, String policy, Step.StepKey currentStepKey, Exception cause, LongSupplier nowSupplier) {
+        this.index = index;
+        this.policy = policy;
+        this.currentStepKey = currentStepKey;
+        this.cause = cause;
+        this.nowSupplier = nowSupplier;
+    }
+
+    Index getIndex() {
+        return index;
+    }
+
+    String getPolicy() {
+        return policy;
+    }
+
+    Step.StepKey getCurrentStepKey() {
+        return currentStepKey;
+    }
+
+    Exception getCause() {
+        return cause;
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) throws IOException {
+        IndexMetaData idxMeta = currentState.getMetaData().index(index);
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            return currentState;
+        }
+        Settings indexSettings = idxMeta.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(idxMeta);
+        if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
+            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
+            return IndexLifecycleRunner.moveClusterStateToErrorStep(index, currentState, currentStepKey, cause, nowSupplier);
+        } else {
+            // either the policy has changed or the step is now
+            // not the same as when we submitted the update task. In
+            // either case we don't want to do anything now
+            return currentState;
+        }
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        throw new ElasticsearchException("policy [" + policy + "] for index [" + index.getName()
+                + "] failed trying to move from step [" + currentStepKey + "] to the ERROR step.", e);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTask.java
new file mode 100644
index 0000000000000..750fd1af5da42
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTask.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+
+import java.util.function.Consumer;
+import java.util.function.LongSupplier;
+
+public class MoveToNextStepUpdateTask extends ClusterStateUpdateTask {
+    private static final Logger logger = LogManager.getLogger(MoveToNextStepUpdateTask.class);
+
+    private final Index index;
+    private final String policy;
+    private final Step.StepKey currentStepKey;
+    private final Step.StepKey nextStepKey;
+    private final LongSupplier nowSupplier;
+    private final Consumer<ClusterState> stateChangeConsumer;
+
+    public MoveToNextStepUpdateTask(Index index, String policy, Step.StepKey currentStepKey, Step.StepKey nextStepKey,
+                                    LongSupplier nowSupplier, Consumer<ClusterState> stateChangeConsumer) {
+        this.index = index;
+        this.policy = policy;
+        this.currentStepKey = currentStepKey;
+        this.nextStepKey = nextStepKey;
+        this.nowSupplier = nowSupplier;
+        this.stateChangeConsumer = stateChangeConsumer;
+    }
+
+    Index getIndex() {
+        return index;
+    }
+
+    String getPolicy() {
+        return policy;
+    }
+
+    Step.StepKey getCurrentStepKey() {
+        return currentStepKey;
+    }
+
+    Step.StepKey getNextStepKey() {
+        return nextStepKey;
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) {
+        IndexMetaData indexMetaData = currentState.getMetaData().index(index);
+        if (indexMetaData == null) {
+            // Index must have been since deleted, ignore it
+            return currentState;
+        }
+        Settings indexSettings = indexMetaData.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(currentState.getMetaData().index(index));
+        if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
+            && currentStepKey.equals(IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
+            logger.trace("moving [{}] to next step ({})", index.getName(), nextStepKey);
+            return IndexLifecycleRunner.moveClusterStateToNextStep(index, currentState, currentStepKey, nextStepKey, nowSupplier);
+        } else {
+            // either the policy has changed or the step is now
+            // not the same as when we submitted the update task. In
+            // either case we don't want to do anything now
+            return currentState;
+        }
+    }
+
+    @Override
+    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+        if (oldState.equals(newState) == false) {
+            stateChangeConsumer.accept(newState);
+        }
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        throw new ElasticsearchException("policy [" + policy + "] for index [" + index.getName() + "] failed trying to move from step ["
+                + currentStepKey + "] to step [" + nextStepKey + "].", e);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTask.java
new file mode 100644
index 0000000000000..0cf24300831cd
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTask.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+
+public class OperationModeUpdateTask extends ClusterStateUpdateTask {
+    private static final Logger logger = LogManager.getLogger(OperationModeUpdateTask.class);
+    private final OperationMode mode;
+
+    public OperationModeUpdateTask(OperationMode mode) {
+        this.mode = mode;
+    }
+
+    OperationMode getOperationMode() {
+        return mode;
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) {
+        IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        if (currentMetadata != null && currentMetadata.getOperationMode().isValidChange(mode) == false) {
+            return currentState;
+        } else if (currentMetadata == null) {
+            currentMetadata = IndexLifecycleMetadata.EMPTY;
+        }
+
+        final OperationMode newMode;
+        if (currentMetadata.getOperationMode().isValidChange(mode)) {
+            newMode = mode;
+        } else {
+            newMode = currentMetadata.getOperationMode();
+        }
+
+        ClusterState.Builder builder = new ClusterState.Builder(currentState);
+        MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
+        metadataBuilder.putCustom(IndexLifecycleMetadata.TYPE,
+            new IndexLifecycleMetadata(currentMetadata.getPolicyMetadatas(), newMode));
+        builder.metaData(metadataBuilder.build());
+        return builder.build();
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        logger.error("unable to update lifecycle metadata with new mode [" + mode + "]", e);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java
new file mode 100644
index 0000000000000..d753a5035f756
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistry.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+public class PolicyStepsRegistry {
+    private static final Logger logger = LogManager.getLogger(PolicyStepsRegistry.class);
+
+    private final Client client;
+    // keeps track of existing policies in the cluster state
+    private final SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap;
+    // keeps track of what the first step in a policy is, the key is policy name
+    private final Map<String, Step> firstStepMap;
+    // keeps track of a mapping from policy/step-name to respective Step, the key is policy name
+    private final Map<String, Map<Step.StepKey, Step>> stepMap;
+    private final NamedXContentRegistry xContentRegistry;
+
+    public PolicyStepsRegistry(NamedXContentRegistry xContentRegistry, Client client) {
+        this(new TreeMap<>(), new HashMap<>(), new HashMap<>(), xContentRegistry, client);
+    }
+
+    PolicyStepsRegistry(SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap,
+                        Map<String, Step> firstStepMap, Map<String, Map<Step.StepKey, Step>> stepMap,
+                        NamedXContentRegistry xContentRegistry, Client client) {
+        this.lifecyclePolicyMap = lifecyclePolicyMap;
+        this.firstStepMap = firstStepMap;
+        this.stepMap = stepMap;
+        this.xContentRegistry = xContentRegistry;
+        this.client = client;
+    }
+
+    SortedMap<String, LifecyclePolicyMetadata> getLifecyclePolicyMap() {
+        return lifecyclePolicyMap;
+    }
+
+    Map<String, Step> getFirstStepMap() {
+        return firstStepMap;
+    }
+
+    Map<String, Map<Step.StepKey, Step>> getStepMap() {
+        return stepMap;
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public void update(ClusterState clusterState) {
+        final IndexLifecycleMetadata meta = clusterState.metaData().custom(IndexLifecycleMetadata.TYPE);
+
+        assert meta != null : "IndexLifecycleMetadata cannot be null when updating the policy steps registry";
+
+        Diff<Map<String, LifecyclePolicyMetadata>> diff = DiffableUtils.diff(lifecyclePolicyMap, meta.getPolicyMetadatas(),
+            DiffableUtils.getStringKeySerializer(),
+            // Use a non-diffable value serializer. Otherwise actions in the same
+            // action and phase that are changed show up as diffs instead of upserts.
+            // We want to treat any change in the policy as an upsert so the map is
+            // correctly rebuilt
+            new DiffableUtils.NonDiffableValueSerializer<String, LifecyclePolicyMetadata>() {
+                @Override
+                public void write(LifecyclePolicyMetadata value, StreamOutput out) {
+                    // This is never called
+                    throw new UnsupportedOperationException("should never be called");
+                }
+
+                @Override
+                public LifecyclePolicyMetadata read(StreamInput in, String key) {
+                    // This is never called
+                    throw new UnsupportedOperationException("should never be called");
+                }
+            });
+        DiffableUtils.MapDiff<String, LifecyclePolicyMetadata, DiffableUtils.KeySerializer<String>> mapDiff = (DiffableUtils.MapDiff) diff;
+
+        for (String deletedPolicyName : mapDiff.getDeletes()) {
+            lifecyclePolicyMap.remove(deletedPolicyName);
+            firstStepMap.remove(deletedPolicyName);
+            stepMap.remove(deletedPolicyName);
+        }
+
+        if (mapDiff.getUpserts().isEmpty() == false) {
+            for (LifecyclePolicyMetadata policyMetadata : mapDiff.getUpserts().values()) {
+                LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client, ClientHelper.INDEX_LIFECYCLE_ORIGIN,
+                        policyMetadata.getHeaders());
+                lifecyclePolicyMap.put(policyMetadata.getName(), policyMetadata);
+                List<Step> policyAsSteps = policyMetadata.getPolicy().toSteps(policyClient);
+                if (policyAsSteps.isEmpty() == false) {
+                    firstStepMap.put(policyMetadata.getName(), policyAsSteps.get(0));
+                    final Map<Step.StepKey, Step> stepMapForPolicy = new HashMap<>();
+                    for (Step step : policyAsSteps) {
+                        assert ErrorStep.NAME.equals(step.getKey().getName()) == false : "unexpected error step in policy";
+                        stepMapForPolicy.put(step.getKey(), step);
+                    }
+                    logger.trace("updating cached steps for [{}] policy, new steps: {}",
+                        policyMetadata.getName(), stepMapForPolicy.keySet());
+                    stepMap.put(policyMetadata.getName(), stepMapForPolicy);
+                }
+            }
+        }
+    }
+
+    private List<Step> parseStepsFromPhase(String policy, String currentPhase, String phaseDef) throws IOException {
+        final PhaseExecutionInfo phaseExecutionInfo;
+        LifecyclePolicyMetadata policyMetadata = lifecyclePolicyMap.get(policy);
+        if (policyMetadata == null) {
+            throw new IllegalStateException("unable to parse steps for policy [" + policy + "] as it doesn't exist");
+        }
+        LifecyclePolicy currentPolicy = policyMetadata.getPolicy();
+        final LifecyclePolicy policyToExecute;
+        if (InitializePolicyContextStep.INITIALIZATION_PHASE.equals(phaseDef)
+            || TerminalPolicyStep.COMPLETED_PHASE.equals(phaseDef)) {
+            // It is ok to re-use potentially modified policy here since we are in an initialization or completed phase
+            policyToExecute = currentPolicy;
+        } else {
+            // if the current phase definition describes an internal step/phase, do not parse
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) {
+                phaseExecutionInfo = PhaseExecutionInfo.parse(parser, currentPhase);
+            }
+            Map<String, Phase> phaseMap = new HashMap<>(currentPolicy.getPhases());
+            if (phaseExecutionInfo.getPhase() != null) {
+                phaseMap.put(currentPhase, phaseExecutionInfo.getPhase());
+            }
+            policyToExecute = new LifecyclePolicy(currentPolicy.getType(), currentPolicy.getName(), phaseMap);
+        }
+        LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client,
+            ClientHelper.INDEX_LIFECYCLE_ORIGIN, lifecyclePolicyMap.get(policy).getHeaders());
+        final List<Step> steps = policyToExecute.toSteps(policyClient);
+        // Build a list of steps that correspond with the phase the index is currently in
+        final List<Step> phaseSteps;
+        if (steps == null) {
+            phaseSteps = new ArrayList<>();
+        } else {
+            phaseSteps = steps.stream()
+                .filter(e -> e.getKey().getPhase().equals(currentPhase))
+                .collect(Collectors.toList());
+        }
+        logger.trace("parsed steps for policy [{}] in phase [{}], definition: [{}], steps: [{}]",
+            policy, currentPhase, phaseDef, phaseSteps);
+        return phaseSteps;
+    }
+
+    @Nullable
+    public Step getStep(final IndexMetaData indexMetaData, final Step.StepKey stepKey) {
+        if (ErrorStep.NAME.equals(stepKey.getName())) {
+            return new ErrorStep(new Step.StepKey(stepKey.getPhase(), stepKey.getAction(), ErrorStep.NAME));
+        }
+
+        final String phase = stepKey.getPhase();
+        final String policyName = indexMetaData.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
+        final Index index = indexMetaData.getIndex();
+
+        if (policyName == null) {
+            throw new IllegalArgumentException("failed to retrieve step " + stepKey + " as index [" + index.getName() + "] has no policy");
+        }
+
+        // parse phase steps from the phase definition in the index settings
+        final String phaseJson = Optional.ofNullable(LifecycleExecutionState.fromIndexMetadata(indexMetaData).getPhaseDefinition())
+            .orElse(InitializePolicyContextStep.INITIALIZATION_PHASE);
+
+        final List<Step> phaseSteps;
+        try {
+            phaseSteps = parseStepsFromPhase(policyName, phase, phaseJson);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to load cached steps for " + stepKey, e);
+        } catch (XContentParseException parseErr) {
+            throw new XContentParseException(parseErr.getLocation(),
+                "failed to load steps for " + stepKey + " from [" + phaseJson + "]", parseErr);
+        }
+
+        assert phaseSteps.stream().allMatch(step -> step.getKey().getPhase().equals(phase)) :
+            "expected phase steps loaded from phase definition for [" + index.getName() + "] to be in phase [" + phase +
+                "] but they were not, steps: " + phaseSteps;
+
+        // Return the step that matches the given stepKey or else null if we couldn't find it
+        return phaseSteps.stream().filter(step -> step.getKey().equals(stepKey)).findFirst().orElse(null);
+    }
+
+    /**
+     * Given a policy and stepkey, return true if a step exists, false otherwise
+     */
+    public boolean stepExists(final String policy, final Step.StepKey stepKey) {
+        Map<Step.StepKey, Step> steps = stepMap.get(policy);
+        if (steps == null) {
+            return false;
+        } else {
+            return steps.containsKey(stepKey);
+        }
+    }
+
+    public boolean policyExists(final String policy) {
+        return lifecyclePolicyMap.containsKey(policy);
+    }
+
+    public Step getFirstStep(String policy) {
+        return firstStepMap.get(policy);
+    }
+
+    public TimeValue getIndexAgeForPhase(final String policy, final String phase) {
+        // These built in phases should never wait
+        if (InitializePolicyContextStep.INITIALIZATION_PHASE.equals(phase) || TerminalPolicyStep.COMPLETED_PHASE.equals(phase)) {
+            return TimeValue.ZERO;
+        }
+        final LifecyclePolicyMetadata meta = lifecyclePolicyMap.get(policy);
+        if (meta == null) {
+            throw new IllegalArgumentException("no policy found with name \"" + policy + "\"");
+        } else {
+            final Phase retrievedPhase = meta.getPolicy().getPhases().get(phase);
+            if (retrievedPhase == null) {
+                // We don't have that phase registered, proceed right through it
+                return TimeValue.ZERO;
+            } else {
+                return retrievedPhase.getMinimumAge();
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTask.java
new file mode 100644
index 0000000000000..72c7aa81b9d77
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTask.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class SetStepInfoUpdateTask extends ClusterStateUpdateTask {
+    private final Index index;
+    private final String policy;
+    private final Step.StepKey currentStepKey;
+    private ToXContentObject stepInfo;
+
+    public SetStepInfoUpdateTask(Index index, String policy, Step.StepKey currentStepKey, ToXContentObject stepInfo) {
+        this.index = index;
+        this.policy = policy;
+        this.currentStepKey = currentStepKey;
+        this.stepInfo = stepInfo;
+    }
+
+    Index getIndex() {
+        return index;
+    }
+
+    String getPolicy() {
+        return policy;
+    }
+
+    Step.StepKey getCurrentStepKey() {
+        return currentStepKey;
+    }
+
+    ToXContentObject getStepInfo() {
+        return stepInfo;
+    }
+
+    @Override
+    public ClusterState execute(ClusterState currentState) throws IOException {
+        IndexMetaData idxMeta = currentState.getMetaData().index(index);
+        if (idxMeta == null) {
+            // Index must have been since deleted, ignore it
+            return currentState;
+        }
+        Settings indexSettings = idxMeta.getSettings();
+        LifecycleExecutionState indexILMData = LifecycleExecutionState.fromIndexMetadata(idxMeta);
+        if (policy.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings))
+                && Objects.equals(currentStepKey, IndexLifecycleRunner.getCurrentStepKey(indexILMData))) {
+            return IndexLifecycleRunner.addStepInfoToClusterState(index, currentState, stepInfo);
+        } else {
+            // either the policy has changed or the step is now
+            // not the same as when we submitted the update task. In
+            // either case we don't want to do anything now
+            return currentState;
+        }
+    }
+
+    @Override
+    public void onFailure(String source, Exception e) {
+        throw new ElasticsearchException("policy [" + policy + "] for index [" + index.getName()
+                + "] failed trying to set step info for step [" + currentStepKey + "].", e);
+    }
+
+    public static class ExceptionWrapper implements ToXContentObject {
+        private final Throwable exception;
+
+        public ExceptionWrapper(Throwable exception) {
+            this.exception = exception;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            ElasticsearchException.generateThrowableXContent(builder, params, exception);
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/TimeValueSchedule.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/TimeValueSchedule.java
new file mode 100644
index 0000000000000..436f8637a0228
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/TimeValueSchedule.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.core.scheduler.SchedulerEngine.Schedule;
+
+import java.util.Objects;
+
+public class TimeValueSchedule implements Schedule {
+
+    private TimeValue interval;
+
+    public TimeValueSchedule(TimeValue interval) {
+        if (interval.millis() <= 0) {
+            throw new IllegalArgumentException("interval must be greater than 0 milliseconds");
+        }
+        this.interval = interval;
+    }
+
+    public TimeValue getInterval() {
+        return interval;
+    }
+
+    @Override
+    public long nextScheduledTimeAfter(long startTime, long time) {
+        assert time >= startTime;
+        if (startTime == time) {
+            time++;
+        }
+        long delta = time - startTime;
+        return startTime + (delta / interval.millis() + 1) * interval.millis();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(interval);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        TimeValueSchedule other = (TimeValueSchedule) obj;
+        return Objects.equals(interval, other.interval);
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestDeleteLifecycleAction.java
new file mode 100644
index 0000000000000..081e7d1565f79
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestDeleteLifecycleAction.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
+
+import java.io.IOException;
+
+public class RestDeleteLifecycleAction extends BaseRestHandler {
+
+    public RestDeleteLifecycleAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.DELETE, "/_ilm/policy/{name}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_delete_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String lifecycleName = restRequest.param("name");
+        DeleteLifecycleAction.Request deleteLifecycleRequest = new DeleteLifecycleAction.Request(lifecycleName);
+        deleteLifecycleRequest.timeout(restRequest.paramAsTime("timeout", deleteLifecycleRequest.timeout()));
+        deleteLifecycleRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", deleteLifecycleRequest.masterNodeTimeout()));
+
+        return channel -> client.execute(DeleteLifecycleAction.INSTANCE, deleteLifecycleRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestExplainLifecycleAction.java
new file mode 100644
index 0000000000000..96be5f0fc0337
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestExplainLifecycleAction.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+
+import java.io.IOException;
+
+public class RestExplainLifecycleAction extends BaseRestHandler {
+
+    public RestExplainLifecycleAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, "/{index}/_ilm/explain", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_explain_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index"));
+        ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest();
+        explainLifecycleRequest.indices(indexes);
+        explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
+        String masterNodeTimeout = restRequest.param("master_timeout");
+        if (masterNodeTimeout != null) {
+            explainLifecycleRequest.masterNodeTimeout(masterNodeTimeout);
+        }
+
+        return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetLifecycleAction.java
new file mode 100644
index 0000000000000..b518fe2f08698
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetLifecycleAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+
+import java.io.IOException;
+
+public class RestGetLifecycleAction extends BaseRestHandler {
+
+    public RestGetLifecycleAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, "/_ilm/policy", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_ilm/policy/{name}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_get_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String[] lifecycleNames = Strings.splitStringByCommaToArray(restRequest.param("name"));
+        GetLifecycleAction.Request getLifecycleRequest = new GetLifecycleAction.Request(lifecycleNames);
+        getLifecycleRequest.timeout(restRequest.paramAsTime("timeout", getLifecycleRequest.timeout()));
+        getLifecycleRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", getLifecycleRequest.masterNodeTimeout()));
+
+        return channel -> client.execute(GetLifecycleAction.INSTANCE, getLifecycleRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetStatusAction.java
new file mode 100644
index 0000000000000..be2d16ee0be76
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestGetStatusAction.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction;
+
+public class RestGetStatusAction extends BaseRestHandler {
+
+    public RestGetStatusAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, "/_ilm/status", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_get_operation_mode_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        GetStatusAction.Request request = new GetStatusAction.Request();
+        request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
+        return channel -> client.execute(GetStatusAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestMoveToStepAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestMoveToStepAction.java
new file mode 100644
index 0000000000000..41228041679e7
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestMoveToStepAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction;
+
+import java.io.IOException;
+
+public class RestMoveToStepAction extends BaseRestHandler {
+
+    public RestMoveToStepAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST,"/_ilm/move/{name}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_move_to_step_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String index = restRequest.param("name");
+        XContentParser parser = restRequest.contentParser();
+        MoveToStepAction.Request request = MoveToStepAction.Request.parseRequest(index, parser);
+        request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
+        return channel -> client.execute(MoveToStepAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestPutLifecycleAction.java
new file mode 100644
index 0000000000000..586c3c683264e
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestPutLifecycleAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+
+import java.io.IOException;
+
+public class RestPutLifecycleAction extends BaseRestHandler {
+
+    public RestPutLifecycleAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.PUT, "/_ilm/policy/{name}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_put_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String lifecycleName = restRequest.param("name");
+        XContentParser parser = restRequest.contentParser();
+        PutLifecycleAction.Request putLifecycleRequest = PutLifecycleAction.Request.parseRequest(lifecycleName, parser);
+        putLifecycleRequest.timeout(restRequest.paramAsTime("timeout", putLifecycleRequest.timeout()));
+        putLifecycleRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", putLifecycleRequest.masterNodeTimeout()));
+
+        return channel -> client.execute(PutLifecycleAction.INSTANCE, putLifecycleRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRemoveIndexLifecyclePolicyAction.java
new file mode 100644
index 0000000000000..d077b732341ca
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRemoveIndexLifecyclePolicyAction.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
+
+import java.io.IOException;
+
+public class RestRemoveIndexLifecyclePolicyAction extends BaseRestHandler {
+
+    public RestRemoveIndexLifecyclePolicyAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_ilm/remove", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_remove_policy_for_index_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index"));
+        RemoveIndexLifecyclePolicyAction.Request changePolicyRequest = new RemoveIndexLifecyclePolicyAction.Request(indexes);
+        changePolicyRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", changePolicyRequest.masterNodeTimeout()));
+        changePolicyRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, changePolicyRequest.indicesOptions()));
+
+        return channel ->
+                client.execute(RemoveIndexLifecyclePolicyAction.INSTANCE, changePolicyRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRetryAction.java
new file mode 100644
index 0000000000000..9e12c3cc34ed7
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestRetryAction.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ *
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
+
+public class RestRetryAction extends BaseRestHandler {
+
+    public RestRetryAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_ilm/retry", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_retry_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        String[] indices = Strings.splitStringByCommaToArray(restRequest.param("index"));
+        RetryAction.Request request = new RetryAction.Request(indices);
+        request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
+        request.indices(indices);
+        request.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen()));
+        return channel -> client.execute(RetryAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStartILMAction.java
new file mode 100644
index 0000000000000..84f46a30406fd
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStartILMAction.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
+
+public class RestStartILMAction extends BaseRestHandler {
+
+    public RestStartILMAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, "/_ilm/start", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_start_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        StartILMRequest request = new StartILMRequest();
+        request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
+        return channel -> client.execute(StartILMAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStopAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStopAction.java
new file mode 100644
index 0000000000000..2f8d3c5e43037
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/RestStopAction.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
+
+public class RestStopAction extends BaseRestHandler {
+
+    public RestStopAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, "/_ilm/stop", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ilm_stop_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+        StopILMRequest request = new StopILMRequest();
+        request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
+        request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
+        return channel -> client.execute(StopILMAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java
new file mode 100644
index 0000000000000..a8b718bbb7be7
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Response;
+
+import java.util.Iterator;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Request, Response> {
+
+    @Inject
+    public TransportDeleteLifecycleAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                          ThreadPool threadPool, ActionFilters actionFilters,
+                                          IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, DeleteLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                indexNameExpressionResolver, Request::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+        clusterService.submitStateUpdateTask("delete-lifecycle-" + request.getPolicyName(),
+                new AckedClusterStateUpdateTask<Response>(request, listener) {
+                    @Override
+                    protected Response newResponse(boolean acknowledged) {
+                        return new Response(acknowledged);
+                    }
+
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        Iterator<IndexMetaData> indicesIt = currentState.metaData().indices().valuesIt();
+                        while(indicesIt.hasNext()) {
+                            IndexMetaData idxMeta = indicesIt.next();
+                            String indexPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
+                            if (request.getPolicyName().equals(indexPolicy)) {
+                                throw new IllegalArgumentException("Cannot delete policy [" + request.getPolicyName()
+                                    + "]. It is being used by at least one index [" + idxMeta.getIndex().getName() + "]");
+                            }
+
+                        }
+                        ClusterState.Builder newState = ClusterState.builder(currentState);
+                        IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);
+                        if (currentMetadata == null
+                                || currentMetadata.getPolicyMetadatas().containsKey(request.getPolicyName()) == false) {
+                            throw new ResourceNotFoundException("Lifecycle policy not found: {}", request.getPolicyName());
+                        }
+                        SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
+                        newPolicies.remove(request.getPolicyName());
+                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+                        newState.metaData(MetaData.builder(currentState.getMetaData())
+                                .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
+                        return newState.build();
+                    }
+                });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportExplainLifecycleAction.java
new file mode 100644
index 0000000000000..addef8a285df9
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportExplainLifecycleAction.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.info.TransportClusterInfoAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleExplainResponse;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TransportExplainLifecycleAction
+        extends TransportClusterInfoAction<ExplainLifecycleRequest, ExplainLifecycleResponse> {
+
+    private final NamedXContentRegistry xContentRegistry;
+
+    @Inject
+    public TransportExplainLifecycleAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                           ThreadPool threadPool, ActionFilters actionFilters,
+                                           IndexNameExpressionResolver indexNameExpressionResolver,
+                                           NamedXContentRegistry xContentRegistry) {
+        super(settings, ExplainLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                indexNameExpressionResolver, ExplainLifecycleRequest::new);
+        this.xContentRegistry = xContentRegistry;
+    }
+
+    @Override
+    protected ExplainLifecycleResponse newResponse() {
+        return new ExplainLifecycleResponse();
+    }
+
+    @Override
+    protected String executor() {
+        // very lightweight operation, no need to fork
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(ExplainLifecycleRequest request, ClusterState state) {
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, request));
+    }
+
+    @Override
+    protected void doMasterOperation(ExplainLifecycleRequest request, String[] concreteIndices, ClusterState state,
+            ActionListener<ExplainLifecycleResponse> listener) {
+        Map<String, IndexLifecycleExplainResponse> indexReponses = new HashMap<>();
+        for (String index : concreteIndices) {
+            IndexMetaData idxMetadata = state.metaData().index(index);
+            Settings idxSettings = idxMetadata.getSettings();
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMetadata);
+            String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxSettings);
+            String currentPhase = lifecycleState.getPhase();
+            String stepInfo = lifecycleState.getStepInfo();
+            BytesArray stepInfoBytes = null;
+            if (stepInfo != null) {
+                stepInfoBytes = new BytesArray(stepInfo);
+            }
+            // parse existing phase steps from the phase definition in the index settings
+            String phaseDef = lifecycleState.getPhaseDefinition();
+            PhaseExecutionInfo phaseExecutionInfo = null;
+            if (Strings.isNullOrEmpty(phaseDef) == false) {
+                try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) {
+                    phaseExecutionInfo = PhaseExecutionInfo.parse(parser, currentPhase);
+                } catch (IOException e) {
+                    listener.onFailure(new ElasticsearchParseException(
+                        "failed to parse phase definition for index [" + index + "]", e));
+                    return;
+                }
+            }
+            final IndexLifecycleExplainResponse indexResponse;
+            if (Strings.hasLength(policyName)) {
+                indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(index, policyName,
+                    lifecycleState.getLifecycleDate(),
+                    lifecycleState.getPhase(),
+                    lifecycleState.getAction(),
+                    lifecycleState.getStep(),
+                    lifecycleState.getFailedStep(),
+                    lifecycleState.getPhaseTime(),
+                    lifecycleState.getActionTime(),
+                    lifecycleState.getStepTime(),
+                    stepInfoBytes,
+                    phaseExecutionInfo);
+            } else {
+                indexResponse = IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index);
+            }
+            indexReponses.put(indexResponse.getIndex(), indexResponse);
+        }
+        listener.onResponse(new ExplainLifecycleResponse(indexReponses));
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java
new file mode 100644
index 0000000000000..c7660dc68d6ab
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.LifecyclePolicyResponseItem;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction.Response;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class TransportGetLifecycleAction extends TransportMasterNodeAction<Request, Response> {
+
+    @Inject
+    public TransportGetLifecycleAction(Settings settings, TransportService transportService, ClusterService clusterService,
+            ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, GetLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                Request::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+        IndexLifecycleMetadata metadata = clusterService.state().metaData().custom(IndexLifecycleMetadata.TYPE);
+        if (metadata == null) {
+            if (request.getPolicyNames().length == 0) {
+                listener.onResponse(new Response(Collections.emptyList()));
+            } else {
+                listener.onFailure(new ResourceNotFoundException("Lifecycle policy not found: {}",
+                    Arrays.toString(request.getPolicyNames())));
+            }
+        } else {
+            List<LifecyclePolicyResponseItem> requestedPolicies;
+            // if no policies explicitly provided, behave as if `*` was specified
+            if (request.getPolicyNames().length == 0) {
+                requestedPolicies = new ArrayList<>(metadata.getPolicyMetadatas().size());
+                for (LifecyclePolicyMetadata policyMetadata : metadata.getPolicyMetadatas().values()) {
+                    requestedPolicies.add(new LifecyclePolicyResponseItem(policyMetadata.getPolicy(),
+                        policyMetadata.getVersion(), policyMetadata.getModifiedDateString()));
+                }
+            } else {
+                requestedPolicies = new ArrayList<>(request.getPolicyNames().length);
+                for (String name : request.getPolicyNames()) {
+                    LifecyclePolicyMetadata policyMetadata = metadata.getPolicyMetadatas().get(name);
+                    if (policyMetadata == null) {
+                        listener.onFailure(new ResourceNotFoundException("Lifecycle policy not found: {}", name));
+                        return;
+                    }
+                    requestedPolicies.add(new LifecyclePolicyResponseItem(policyMetadata.getPolicy(),
+                        policyMetadata.getVersion(), policyMetadata.getModifiedDateString()));
+                }
+            }
+            listener.onResponse(new Response(requestedPolicies));
+        }
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java
new file mode 100644
index 0000000000000..b5f777540aa04
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction.Response;
+
+public class TransportGetStatusAction extends TransportMasterNodeAction<Request, Response> {
+
+    @Inject
+    public TransportGetStatusAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                    ThreadPool threadPool, ActionFilters actionFilters,
+                                    IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, GetStatusAction.NAME, transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver, Request::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+        IndexLifecycleMetadata metadata = state.metaData().custom(IndexLifecycleMetadata.TYPE);
+        final Response response;
+        if (metadata == null) {
+            // no need to actually install metadata just yet, but safe to say it is not stopped
+            response = new Response(OperationMode.RUNNING);
+        } else {
+            response = new Response(metadata.getOperationMode());
+        }
+        listener.onResponse(response);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java
new file mode 100644
index 0000000000000..a2744569e213c
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Response;
+import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleService;
+
+public class TransportMoveToStepAction extends TransportMasterNodeAction<Request, Response> {
+    IndexLifecycleService indexLifecycleService;
+    @Inject
+    public TransportMoveToStepAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                     ThreadPool threadPool, ActionFilters actionFilters,
+                                     IndexNameExpressionResolver indexNameExpressionResolver,
+                                     IndexLifecycleService indexLifecycleService) {
+        super(settings, MoveToStepAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                Request::new);
+        this.indexLifecycleService = indexLifecycleService;
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+        IndexMetaData indexMetaData = state.metaData().index(request.getIndex());
+        if (indexMetaData == null) {
+            listener.onFailure(new IllegalArgumentException("index [" + request.getIndex() + "] does not exist"));
+            return;
+        }
+        clusterService.submitStateUpdateTask("index[" + request.getIndex() + "]-move-to-step",
+            new AckedClusterStateUpdateTask<Response>(request, listener) {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return indexLifecycleService.moveClusterStateToStep(currentState, request.getIndex(), request.getCurrentStepKey(),
+                        request.getNextStepKey());
+                }
+
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    IndexMetaData newIndexMetaData = newState.metaData().index(indexMetaData.getIndex());
+                    if (newIndexMetaData == null) {
+                        // The index has somehow been deleted - there shouldn't be any opportunity for this to happen, but just in case.
+                        logger.debug("index [" + indexMetaData.getIndex() + "] has been deleted after moving to step [" +
+                            request.getNextStepKey() + "], skipping async action check");
+                        return;
+                    }
+                    indexLifecycleService.maybeRunAsyncAction(newState, newIndexMetaData, request.getNextStepKey());
+                }
+
+                @Override
+                protected Response newResponse(boolean acknowledged) {
+                    return new Response(acknowledged);
+                }
+            });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java
new file mode 100644
index 0000000000000..2a56f179f39a5
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
+
+import java.time.Instant;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/**
+ * This class is responsible for bootstrapping {@link IndexLifecycleMetadata} into the cluster-state, as well
+ * as adding the desired new policy to be inserted.
+ */
+public class TransportPutLifecycleAction extends TransportMasterNodeAction<Request, Response> {
+
+    @Inject
+    public TransportPutLifecycleAction(Settings settings, TransportService transportService, ClusterService clusterService,
+            ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, PutLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                Request::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+        // headers from the thread context stored by the AuthenticationService to be shared between the
+        // REST layer and the Transport layer here must be accessed within this thread and not in the
+        // cluster state thread in the ClusterStateUpdateTask below since that thread does not share the
+        // same context, and therefore does not have access to the appropriate security headers.
+        Map<String, String> filteredHeaders = threadPool.getThreadContext().getHeaders().entrySet().stream()
+            .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey()))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        clusterService.submitStateUpdateTask("put-lifecycle-" + request.getPolicy().getName(),
+                new AckedClusterStateUpdateTask<Response>(request, listener) {
+                    @Override
+                    protected Response newResponse(boolean acknowledged) {
+                        return new Response(acknowledged);
+                    }
+
+                    @Override
+                    public ClusterState execute(ClusterState currentState) throws Exception {
+                        ClusterState.Builder newState = ClusterState.builder(currentState);
+                        IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);
+                        if (currentMetadata == null) { // first time using index-lifecycle feature, bootstrap metadata
+                            currentMetadata = IndexLifecycleMetadata.EMPTY;
+                        }
+                        LifecyclePolicyMetadata existingPolicyMetadata = currentMetadata.getPolicyMetadatas()
+                            .get(request.getPolicy().getName());
+                        long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L;
+                        SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
+                        LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders,
+                            nextVersion, Instant.now().toEpochMilli());
+                        newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
+                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, OperationMode.RUNNING);
+                        newState.metaData(MetaData.builder(currentState.getMetaData())
+                                .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
+                        return newState.build();
+                    }
+                });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java
new file mode 100644
index 0000000000000..149921a29a3d1
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction.Response;
+import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleRunner;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TransportRemoveIndexLifecyclePolicyAction extends TransportMasterNodeAction<Request, Response> {
+
+    @Inject
+    public TransportRemoveIndexLifecyclePolicyAction(Settings settings, TransportService transportService, ClusterService clusterService,
+            ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, RemoveIndexLifecyclePolicyAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                indexNameExpressionResolver, Request::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+        final Index[] indices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), request.indices());
+        clusterService.submitStateUpdateTask("remove-lifecycle-for-index",
+                new AckedClusterStateUpdateTask<Response>(request, listener) {
+
+                    private final List<String> failedIndexes = new ArrayList<>();
+
+                    @Override
+                    public ClusterState execute(ClusterState currentState) throws Exception {
+                        return IndexLifecycleRunner.removePolicyForIndexes(indices, currentState, failedIndexes);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        listener.onFailure(e);
+                    }
+
+                    @Override
+                    protected Response newResponse(boolean acknowledged) {
+                        return new Response(failedIndexes);
+                    }
+                });
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java
new file mode 100644
index 0000000000000..4bbefafee0866
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Request;
+import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Response;
+import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleService;
+
+public class TransportRetryAction extends TransportMasterNodeAction<Request, Response> {
+
+    IndexLifecycleService indexLifecycleService;
+
+    @Inject
+    public TransportRetryAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                ThreadPool threadPool, ActionFilters actionFilters,
+                                IndexNameExpressionResolver indexNameExpressionResolver,
+                                IndexLifecycleService indexLifecycleService) {
+        super(settings, RetryAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                Request::new);
+        this.indexLifecycleService = indexLifecycleService;
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected Response newResponse() {
+        return new Response();
+    }
+
+    @Override
+    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+        clusterService.submitStateUpdateTask("ilm-re-run",
+            new AckedClusterStateUpdateTask<Response>(request, listener) {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    return indexLifecycleService.moveClusterStateToFailedStep(currentState, request.indices());
+                }
+
+                @Override
+                protected Response newResponse(boolean acknowledged) {
+                    return new Response(acknowledged);
+                }
+            });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java
new file mode 100644
index 0000000000000..f4abe9fd4c79f
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
+import org.elasticsearch.xpack.indexlifecycle.OperationModeUpdateTask;
+
+public class TransportStartILMAction extends TransportMasterNodeAction<StartILMRequest, AcknowledgedResponse> {
+
+    @Inject
+    public TransportStartILMAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                   ThreadPool threadPool, ActionFilters actionFilters,
+                                   IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, StartILMAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                StartILMRequest::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
+    }
+
+    @Override
+    protected void masterOperation(StartILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+        clusterService.submitStateUpdateTask("ilm_operation_mode_update",
+                new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                        return (new OperationModeUpdateTask(OperationMode.RUNNING)).execute(currentState);
+                }
+
+                @Override
+                    protected AcknowledgedResponse newResponse(boolean acknowledged) {
+                        return new AcknowledgedResponse(acknowledged);
+                }
+            });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(StartILMRequest request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java
new file mode 100644
index 0000000000000..55e036e5f11e0
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
+import org.elasticsearch.xpack.indexlifecycle.OperationModeUpdateTask;
+
+public class TransportStopILMAction extends TransportMasterNodeAction<StopILMRequest, AcknowledgedResponse> {
+
+    @Inject
+    public TransportStopILMAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                  ThreadPool threadPool, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, StopILMAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+                StopILMRequest::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        return new AcknowledgedResponse();
+    }
+
+    @Override
+    protected void masterOperation(StopILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+        clusterService.submitStateUpdateTask("ilm_operation_mode_update",
+                new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                        return (new OperationModeUpdateTask(OperationMode.STOPPING)).execute(currentState);
+                }
+
+                @Override
+                    protected AcknowledgedResponse newResponse(boolean acknowledged) {
+                        return new AcknowledgedResponse(acknowledged);
+                }
+            });
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(StopILMRequest request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTestsUtils.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTestsUtils.java
new file mode 100644
index 0000000000000..3776363cf175e
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/LifecyclePolicyTestsUtils.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.indexlifecycle;
+
+import org.elasticsearch.xpack.indexlifecycle.LockableLifecycleType;
+
+import java.util.Map;
+
+/**
+ * This class is here for constructing instances of {@link LifecyclePolicy} that differs from
+ * the main {@link TimeseriesLifecycleType} one. Since the more generic constructor is package-private so
+ * that users are not exposed to {@link LifecycleType}, it is still useful to construct different ones for
+ * testing purposes
+ */
+public class LifecyclePolicyTestsUtils {
+
+    public static LifecyclePolicy newTestLifecyclePolicy(String policyName, Map<String, Phase> phases) {
+        return new LifecyclePolicy(TestLifecycleType.INSTANCE, policyName, phases);
+    }
+
+    public static LifecyclePolicy newLockableLifecyclePolicy(String policyName, Map<String, Phase> phases) {
+        return new LifecyclePolicy(LockableLifecycleType.INSTANCE, policyName, phases);
+    }
+
+    public static LifecyclePolicy randomTimeseriesLifecyclePolicy(String policyName) {
+        return LifecyclePolicyTests.randomTimeseriesLifecyclePolicy(policyName);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java
new file mode 100644
index 0000000000000..21f2b0e70939f
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/ExecuteStepsUpdateTaskTests.java
@@ -0,0 +1,307 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.MockStep;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleRunnerTests.MockClusterStateActionStep;
+import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleRunnerTests.MockClusterStateWaitStep;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class ExecuteStepsUpdateTaskTests extends ESTestCase {
+
+    private static final StepKey firstStepKey = new StepKey("first_phase", "action_1", "step_1");
+    private static final StepKey secondStepKey = new StepKey("first_phase", "action_1", "step_2");
+    private static final StepKey thirdStepKey = new StepKey("first_phase", "action_1", "step_3");
+    private static final StepKey invalidStepKey = new StepKey("invalid", "invalid", "invalid");
+    private ClusterState clusterState;
+    private PolicyStepsRegistry policyStepsRegistry;
+    private String mixedPolicyName;
+    private String allClusterPolicyName;
+    private String invalidPolicyName;
+    private Index index;
+    private IndexMetaData indexMetaData;
+    private MockClusterStateActionStep firstStep;
+    private MockClusterStateWaitStep secondStep;
+    private MockClusterStateWaitStep allClusterSecondStep;
+    private MockStep thirdStep;
+    private Client client;
+    private IndexLifecycleMetadata lifecycleMetadata;
+    private String indexName;
+
+    @Before
+    public void prepareState() throws IOException {
+        client = Mockito.mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        firstStep = new MockClusterStateActionStep(firstStepKey, secondStepKey);
+        secondStep = new MockClusterStateWaitStep(secondStepKey, thirdStepKey);
+        secondStep.setWillComplete(true);
+        allClusterSecondStep = new MockClusterStateWaitStep(secondStepKey, TerminalPolicyStep.KEY);
+        allClusterSecondStep.setWillComplete(true);
+        thirdStep = new MockStep(thirdStepKey, null);
+        mixedPolicyName = randomAlphaOfLengthBetween(5, 10);
+        allClusterPolicyName = randomAlphaOfLengthBetween(1, 4);
+        invalidPolicyName = randomAlphaOfLength(11);
+        Phase mixedPhase = new Phase("first_phase", TimeValue.ZERO, Collections.singletonMap(MockAction.NAME,
+            new MockAction(Arrays.asList(firstStep, secondStep, thirdStep))));
+        Phase allClusterPhase = new Phase("first_phase", TimeValue.ZERO, Collections.singletonMap(MockAction.NAME,
+            new MockAction(Arrays.asList(firstStep, allClusterSecondStep))));
+        Phase invalidPhase = new Phase("invalid_phase", TimeValue.ZERO, Collections.singletonMap(MockAction.NAME,
+            new MockAction(Arrays.asList(new MockClusterStateActionStep(firstStepKey, invalidStepKey)))));
+        LifecyclePolicy mixedPolicy = newTestLifecyclePolicy(mixedPolicyName,
+            Collections.singletonMap(mixedPhase.getName(), mixedPhase));
+        LifecyclePolicy allClusterPolicy = newTestLifecyclePolicy(allClusterPolicyName,
+            Collections.singletonMap(allClusterPhase.getName(), allClusterPhase));
+        LifecyclePolicy invalidPolicy = newTestLifecyclePolicy(invalidPolicyName,
+            Collections.singletonMap(invalidPhase.getName(), invalidPhase));
+        Map<String, LifecyclePolicyMetadata> policyMap = new HashMap<>();
+        policyMap.put(mixedPolicyName, new LifecyclePolicyMetadata(mixedPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        policyMap.put(allClusterPolicyName, new LifecyclePolicyMetadata(allClusterPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        policyMap.put(invalidPolicyName, new LifecyclePolicyMetadata(invalidPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        policyStepsRegistry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY, client);
+
+        indexName = randomAlphaOfLength(5);
+        lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        indexMetaData = setupIndexPolicy(mixedPolicyName);
+    }
+
+    private IndexMetaData setupIndexPolicy(String policyName) {
+        // Reset the index to use the "allClusterPolicyName"
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("new");
+        lifecycleState.setAction("init");
+        lifecycleState.setStep("init");
+        IndexMetaData indexMetadata = IndexMetaData.builder(indexName)
+            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        index = indexMetadata.getIndex();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+        policyStepsRegistry.update(clusterState);
+        return indexMetadata;
+    }
+
+    public void testNeverExecuteNonClusterStateStep() throws IOException {
+        setStateToKey(thirdStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, thirdStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        assertThat(task.execute(clusterState), sameInstance(clusterState));
+    }
+
+    public void testSuccessThenFailureUnsetNextKey() throws IOException {
+        secondStep.setWillComplete(false);
+        setStateToKey(firstStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, firstStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(secondStepKey));
+        assertThat(firstStep.getExecuteCount(), equalTo(1L));
+        assertThat(secondStep.getExecuteCount(), equalTo(1L));
+        assertThat(task.getNextStepKey(), nullValue());
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), nullValue());
+    }
+
+    public void testExecuteUntilFirstNonClusterStateStep() throws IOException {
+        setStateToKey(secondStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, secondStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(thirdStepKey));
+        assertThat(firstStep.getExecuteCount(), equalTo(0L));
+        assertThat(secondStep.getExecuteCount(), equalTo(1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), nullValue());
+    }
+
+    public void testExecuteInvalidStartStep() throws IOException {
+        // Unset the index's phase/action/step to simulate starting from scratch
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)));
+        lifecycleState.setPhase(null);
+        lifecycleState.setAction(null);
+        lifecycleState.setStep(null);
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.getMetaData())
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
+
+        policyStepsRegistry.update(clusterState);
+
+        Step invalidStep = new MockClusterStateActionStep(firstStepKey, secondStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(invalidPolicyName, index,
+            invalidStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        assertSame(newState, clusterState);
+    }
+
+    public void testExecuteIncompleteWaitStepNoInfo() throws IOException {
+        secondStep.setWillComplete(false);
+        setStateToKey(secondStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, secondStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(secondStepKey));
+        assertThat(firstStep.getExecuteCount(), equalTo(0L));
+        assertThat(secondStep.getExecuteCount(), equalTo(1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), nullValue());
+    }
+
+    public void testExecuteIncompleteWaitStepWithInfo() throws IOException {
+        secondStep.setWillComplete(false);
+        RandomStepInfo stepInfo = new RandomStepInfo(() -> randomAlphaOfLength(10));
+        secondStep.expectedInfo(stepInfo);
+        setStateToKey(secondStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, secondStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(secondStepKey));
+        assertThat(firstStep.getExecuteCount(), equalTo(0L));
+        assertThat(secondStep.getExecuteCount(), equalTo(1L));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), equalTo(stepInfo.toString()));
+    }
+
+    public void testOnFailure() throws IOException {
+        setStateToKey(secondStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, secondStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        Exception expectedException = new RuntimeException();
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> task.onFailure(randomAlphaOfLength(10), expectedException));
+        assertEquals("policy [" + mixedPolicyName + "] for index [" + index.getName() + "] failed on step [" + startStep.getKey() + "].",
+                exception.getMessage());
+        assertSame(expectedException, exception.getCause());
+    }
+
+    public void testClusterActionStepThrowsException() throws IOException {
+        RuntimeException thrownException = new RuntimeException("error");
+        firstStep.setException(thrownException);
+        setStateToKey(firstStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, firstStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(new StepKey(firstStepKey.getPhase(), firstStepKey.getAction(), ErrorStep.NAME)));
+        assertThat(firstStep.getExecuteCount(), equalTo(1L));
+        assertThat(secondStep.getExecuteCount(), equalTo(0L));
+        assertThat(task.getNextStepKey(), equalTo(secondStep.getKey()));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), equalTo("{\"type\":\"runtime_exception\",\"reason\":\"error\"}"));
+    }
+
+    public void testClusterWaitStepThrowsException() throws IOException {
+        RuntimeException thrownException = new RuntimeException("error");
+        secondStep.setException(thrownException);
+        setStateToKey(firstStepKey);
+        Step startStep = policyStepsRegistry.getStep(indexMetaData, firstStepKey);
+        long now = randomNonNegativeLong();
+        ExecuteStepsUpdateTask task = new ExecuteStepsUpdateTask(mixedPolicyName, index, startStep, policyStepsRegistry, null, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey currentStepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(currentStepKey, equalTo(new StepKey(firstStepKey.getPhase(), firstStepKey.getAction(), ErrorStep.NAME)));
+        assertThat(firstStep.getExecuteCount(), equalTo(1L));
+        assertThat(secondStep.getExecuteCount(), equalTo(1L));
+        assertThat(task.getNextStepKey(), equalTo(thirdStepKey));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepInfo(), equalTo("{\"type\":\"runtime_exception\",\"reason\":\"error\"}"));
+    }
+
+    private void setStateToKey(StepKey stepKey) throws IOException {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.getMetaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.getMetaData())
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
+        policyStepsRegistry.update(clusterState);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java
new file mode 100644
index 0000000000000..d83a41b4e60bc
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndexLifecycleFeatureSetTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+    private ClusterService clusterService;
+
+    @Before
+    public void init() throws Exception {
+        licenseState = mock(XPackLicenseState.class);
+        clusterService = mock(ClusterService.class);
+    }
+
+    public void testAvailable() {
+        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+
+        when(licenseState.isIndexLifecycleAllowed()).thenReturn(false);
+        assertThat(featureSet.available(), equalTo(false));
+
+        when(licenseState.isIndexLifecycleAllowed()).thenReturn(true);
+        assertThat(featureSet.available(), equalTo(true));
+
+        featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, null, clusterService);
+        assertThat(featureSet.available(), equalTo(false));
+    }
+
+    public void testEnabled() {
+        Settings.Builder settings = Settings.builder().put("xpack.ilm.enabled", false);
+        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(false));
+
+        settings = Settings.builder().put("xpack.ilm.enabled", true);
+        featureSet = new IndexLifecycleFeatureSet(settings.build(), licenseState, clusterService);
+        assertThat(featureSet.enabled(), equalTo(true));
+    }
+
+    public void testName() {
+        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        assertThat(featureSet.name(), equalTo("ilm"));
+    }
+
+    public void testNativeCodeInfo() {
+        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        assertNull(featureSet.nativeCodeInfo());
+    }
+
+    public void testUsageStats() throws Exception {
+        Map<String, String> indexPolicies = new HashMap<>();
+        List<LifecyclePolicy> policies = new ArrayList<>();
+        String policy1Name = randomAlphaOfLength(10);
+        String policy2Name = randomAlphaOfLength(10);
+        String policy3Name = randomAlphaOfLength(10);
+        indexPolicies.put("index_1", policy1Name);
+        indexPolicies.put("index_2", policy1Name);
+        indexPolicies.put("index_3", policy1Name);
+        indexPolicies.put("index_4", policy1Name);
+        indexPolicies.put("index_5", policy3Name);
+        LifecyclePolicy policy1 = new LifecyclePolicy(policy1Name, Collections.emptyMap());
+        policies.add(policy1);
+        PolicyStats policy1Stats = new PolicyStats(Collections.emptyMap(), 4);
+
+        Map<String, Phase> phases1 = new HashMap<>();
+        LifecyclePolicy policy2 = new LifecyclePolicy(policy2Name, phases1);
+        policies.add(policy2);
+        PolicyStats policy2Stats = new PolicyStats(Collections.emptyMap(), 0);
+
+        LifecyclePolicy policy3 = new LifecyclePolicy(policy3Name, Collections.emptyMap());
+        policies.add(policy3);
+        PolicyStats policy3Stats = new PolicyStats(Collections.emptyMap(), 1);
+
+        ClusterState clusterState = buildClusterState(policies, indexPolicies);
+        Mockito.when(clusterService.state()).thenReturn(clusterState);
+
+        PlainActionFuture<IndexLifecycleFeatureSet.Usage> future = new PlainActionFuture<>();
+        IndexLifecycleFeatureSet ilmFeatureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        ilmFeatureSet.usage(future);
+        IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get();
+        assertThat(ilmUsage.enabled(), equalTo(ilmFeatureSet.enabled()));
+        assertThat(ilmUsage.available(), equalTo(ilmFeatureSet.available()));
+
+        List<PolicyStats> policyStatsList = ilmUsage.getPolicyStats();
+        assertThat(policyStatsList.size(), equalTo(policies.size()));
+        assertTrue(policyStatsList.contains(policy1Stats));
+        assertTrue(policyStatsList.contains(policy2Stats));
+        assertTrue(policyStatsList.contains(policy3Stats));
+
+    }
+
+    private ClusterState buildClusterState(List<LifecyclePolicy> lifecyclePolicies, Map<String, String> indexPolicies) {
+        Map<String, LifecyclePolicyMetadata> lifecyclePolicyMetadatasMap = lifecyclePolicies.stream()
+                .map(p -> new LifecyclePolicyMetadata(p, Collections.emptyMap(), 1, 0L))
+                .collect(Collectors.toMap(LifecyclePolicyMetadata::getName, Function.identity()));
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(lifecyclePolicyMetadatasMap, OperationMode.RUNNING);
+
+        MetaData.Builder metadata = MetaData.builder().putCustom(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata);
+        indexPolicies.forEach((indexName, policyName) -> {
+            Settings indexSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+            IndexMetaData.Builder indexMetadata = IndexMetaData.builder(indexName).settings(indexSettings);
+            metadata.put(indexMetadata);
+        });
+        
+        return ClusterState.builder(new ClusterName("my_cluster")).metaData(metadata).build();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java
new file mode 100644
index 0000000000000..7bd974d31c176
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetUsageTests.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class IndexLifecycleFeatureSetUsageTests extends AbstractWireSerializingTestCase<IndexLifecycleFeatureSetUsage> {
+
+    @Override
+    protected IndexLifecycleFeatureSetUsage createTestInstance() {
+        boolean available = randomBoolean();
+        boolean enabled = randomBoolean();
+        List<PolicyStats> policyStats = new ArrayList<>();
+        int size = randomIntBetween(0, 10);
+        for (int i = 0; i < size; i++) {
+            policyStats.add(PolicyStatsTests.randomPolicyStats());
+        }
+        return new IndexLifecycleFeatureSetUsage(available, enabled, policyStats);
+    }
+
+    @Override
+    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) throws IOException {
+        boolean available = instance.available();
+        boolean enabled = instance.enabled();
+        List<PolicyStats> policyStats = instance.getPolicyStats();
+        switch (between(0, 2)) {
+        case 0:
+            available = available == false;
+            break;
+        case 1:
+            enabled = enabled == false;
+            break;
+        case 2:
+            policyStats = new ArrayList<>(policyStats);
+            policyStats.add(PolicyStatsTests.randomPolicyStats());
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new IndexLifecycleFeatureSetUsage(available, enabled, policyStats);
+    }
+
+    @Override
+    protected Reader<IndexLifecycleFeatureSetUsage> instanceReader() {
+        return IndexLifecycleFeatureSetUsage::new;
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java
new file mode 100644
index 0000000000000..a041232d8a7e7
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java
@@ -0,0 +1,492 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
+import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleExplainResponse;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.client.Requests.clusterHealthRequest;
+import static org.elasticsearch.client.Requests.createIndexRequest;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newLockableLifecyclePolicy;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.core.CombinableMatcher.both;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNull.nullValue;
+
+@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
+    private Settings settings;
+    private LifecyclePolicy lifecyclePolicy;
+    private Phase mockPhase;
+    private static final ObservableAction OBSERVABLE_ACTION;
+    static {
+        List<Step> steps = new ArrayList<>();
+        Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME);
+        steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY));
+        OBSERVABLE_ACTION = new ObservableAction(steps, true);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        settings.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true);
+        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
+        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
+        settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
+        settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
+        return settings.build();
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Settings transportClientSettings() {
+        Settings.Builder settings = Settings.builder().put(super.transportClientSettings());
+        settings.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), true);
+        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
+        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
+        settings.put(XPackSettings.MONITORING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(XPackSettings.LOGSTASH_ENABLED.getKey(), false);
+        return settings.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class, TestILMPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
+    @Before
+    public void init() {
+        settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0).put(LifecycleSettings.LIFECYCLE_NAME, "test").build();
+        List<Step> steps = new ArrayList<>();
+        Step.StepKey key = new Step.StepKey("mock", ObservableAction.NAME, ObservableClusterStateWaitStep.NAME);
+        steps.add(new ObservableClusterStateWaitStep(key, TerminalPolicyStep.KEY));
+        Map<String, LifecycleAction> actions = Collections.singletonMap(ObservableAction.NAME, OBSERVABLE_ACTION);
+        mockPhase = new Phase("mock", TimeValue.timeValueSeconds(0), actions);
+        Map<String, Phase> phases = Collections.singletonMap("mock", mockPhase);
+        lifecyclePolicy = newLockableLifecyclePolicy("test", phases);
+    }
+
+    public void testSingleNodeCluster() throws Exception {
+        settings = Settings.builder().put(settings).put("index.lifecycle.test.complete", true).build();
+        // start master node
+        logger.info("Starting server1");
+        final String server_1 = internalCluster().startNode();
+        final String node1 = getLocalNodeId(server_1);
+
+        // test get-lifecycle behavior when IndexLifecycleMetaData is null
+        GetLifecycleAction.Response getUninitializedLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request()).get();
+        assertThat(getUninitializedLifecycleResponse.getPolicies().size(), equalTo(0));
+        ExecutionException exception = expectThrows(ExecutionException.class,() -> client()
+            .execute(GetLifecycleAction.INSTANCE, new GetLifecycleAction.Request("non-existent-policy")).get());
+        assertThat(exception.getMessage(), containsString("Lifecycle policy not found: [non-existent-policy]"));
+
+        logger.info("Creating lifecycle [test_lifecycle]");
+        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
+        long lowerBoundModifiedDate = Instant.now().toEpochMilli();
+        PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
+        assertAcked(putLifecycleResponse);
+        long upperBoundModifiedDate = Instant.now().toEpochMilli();
+
+        // assert version and modified_date
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request()).get();
+        assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
+        GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
+        assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
+        assertThat(responseItem.getVersion(), equalTo(1L));
+        long actualModifiedDate = Instant.parse(responseItem.getModifiedDate()).toEpochMilli();
+        assertThat(actualModifiedDate,
+            is(both(greaterThanOrEqualTo(lowerBoundModifiedDate)).and(lessThanOrEqualTo(upperBoundModifiedDate))));
+
+        logger.info("Creating index [test]");
+        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test").settings(settings))
+                .actionGet();
+        assertAcked(createIndexResponse);
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        RoutingNode routingNodeEntry1 = clusterState.getRoutingNodes().node(node1);
+        assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(1));
+        assertBusy(() -> {
+            assertEquals(true, client().admin().indices().prepareExists("test").get().isExists());
+        });
+        IndexLifecycleService indexLifecycleService = internalCluster().getInstance(IndexLifecycleService.class, server_1);
+        assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
+        assertNotNull(indexLifecycleService.getScheduledJob());
+        assertBusy(() -> {
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
+        });
+    }
+
+    public void testExplainExecution() throws Exception {
+        // start node
+        logger.info("Starting server1");
+        final String server_1 = internalCluster().startNode();
+        logger.info("Creating lifecycle [test_lifecycle]");
+        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
+        PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
+        assertAcked(putLifecycleResponse);
+
+        GetLifecycleAction.Response getLifecycleResponse = client().execute(GetLifecycleAction.INSTANCE,
+            new GetLifecycleAction.Request()).get();
+        assertThat(getLifecycleResponse.getPolicies().size(), equalTo(1));
+        GetLifecycleAction.LifecyclePolicyResponseItem responseItem = getLifecycleResponse.getPolicies().get(0);
+        assertThat(responseItem.getLifecyclePolicy(), equalTo(lifecyclePolicy));
+        assertThat(responseItem.getVersion(), equalTo(1L));
+        long actualModifiedDate = Instant.parse(responseItem.getModifiedDate()).toEpochMilli();
+
+        logger.info("Creating index [test]");
+        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test").settings(settings))
+            .actionGet();
+        assertAcked(createIndexResponse);
+
+        {
+            PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), mockPhase, 1L, actualModifiedDate);
+            assertBusy(() -> {
+                ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
+                ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
+                assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
+                IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
+                assertThat(indexResponse.getStep(), equalTo("observable_cluster_state_action"));
+                assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
+            });
+        }
+
+        // complete the step
+        client().admin().indices().prepareUpdateSettings("test")
+            .setSettings(Collections.singletonMap("index.lifecycle.test.complete", true)).get();
+
+        {
+            PhaseExecutionInfo expectedExecutionInfo = new PhaseExecutionInfo(lifecyclePolicy.getName(), null, 1L, actualModifiedDate);
+            assertBusy(() -> {
+                ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest();
+                ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get();
+                assertThat(explainResponse.getIndexResponses().size(), equalTo(1));
+                IndexLifecycleExplainResponse indexResponse = explainResponse.getIndexResponses().get("test");
+                assertThat(indexResponse.getPhase(), equalTo(TerminalPolicyStep.COMPLETED_PHASE));
+                assertThat(indexResponse.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
+                assertThat(indexResponse.getPhaseExecutionInfo(), equalTo(expectedExecutionInfo));
+            });
+        }
+    }
+
+    public void testMasterDedicatedDataDedicated() throws Exception {
+        settings = Settings.builder().put(settings).put("index.lifecycle.test.complete", true).build();
+        // start master node
+        logger.info("Starting master-only server1");
+        final String server_1 = internalCluster().startMasterOnlyNode();
+        // start data node
+        logger.info("Starting data-only server2");
+        final String server_2 = internalCluster().startDataOnlyNode();
+        final String node2 = getLocalNodeId(server_2);
+
+        // check that the scheduler was started on the appropriate node
+        {
+            IndexLifecycleService indexLifecycleService = internalCluster().getInstance(IndexLifecycleService.class, server_1);
+            assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
+            assertNotNull(indexLifecycleService.getScheduledJob());
+        }
+        {
+            IndexLifecycleService indexLifecycleService = internalCluster().getInstance(IndexLifecycleService.class, server_2);
+            assertNull(indexLifecycleService.getScheduler());
+            assertNull(indexLifecycleService.getScheduledJob());
+        }
+
+        logger.info("Creating lifecycle [test_lifecycle]");
+        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
+        PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
+        assertAcked(putLifecycleResponse);
+        logger.info("Creating index [test]");
+        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test").settings(settings))
+                .actionGet();
+        assertAcked(createIndexResponse);
+
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        RoutingNode routingNodeEntry1 = clusterState.getRoutingNodes().node(node2);
+        assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(1));
+
+        assertBusy(() -> {
+            assertEquals(true, client().admin().indices().prepareExists("test").get().isExists());
+        });
+        assertBusy(() -> {
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
+        });
+    }
+
+    public void testMasterFailover() throws Exception {
+        // start one server
+        logger.info("Starting sever1");
+        final String server_1 = internalCluster().startNode();
+        final String node1 = getLocalNodeId(server_1);
+
+        logger.info("Creating lifecycle [test_lifecycle]");
+        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
+        PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
+        assertAcked(putLifecycleResponse);
+
+        logger.info("Creating index [test]");
+        CreateIndexResponse createIndexResponse = client().admin().indices().create(createIndexRequest("test").settings(settings))
+                .actionGet();
+        assertAcked(createIndexResponse);
+
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        RoutingNode routingNodeEntry1 = clusterState.getRoutingNodes().node(node1);
+        assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(1));
+
+        logger.info("Starting server2");
+        // start another server
+        internalCluster().startNode();
+
+        // first wait for 2 nodes in the cluster
+        logger.info("Waiting for replicas to be assigned");
+        ClusterHealthResponse clusterHealth = client().admin().cluster()
+                .health(clusterHealthRequest().waitForGreenStatus().waitForNodes("2")).actionGet();
+        logger.info("Done Cluster Health, status {}", clusterHealth.getStatus());
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+        assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
+
+        // check step in progress in lifecycle
+        assertBusy(() -> {
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
+            assertThat(lifecycleState.getStep(), equalTo(ObservableClusterStateWaitStep.NAME));
+        });
+
+        if (randomBoolean()) {
+            // this checks that the phase execution is picked up from the phase definition settings
+            logger.info("updating lifecycle [test_lifecycle] to be empty");
+            PutLifecycleAction.Request updateLifecycleRequest = new PutLifecycleAction.Request
+                (newLockableLifecyclePolicy(lifecyclePolicy.getName(), Collections.emptyMap()));
+            PutLifecycleAction.Response updateLifecycleResponse = client()
+                .execute(PutLifecycleAction.INSTANCE, updateLifecycleRequest).get();
+            assertAcked(updateLifecycleResponse);
+        }
+
+
+        logger.info("Closing server1");
+        // kill the first server
+        internalCluster().stopCurrentMasterNode();
+
+        // check that index lifecycle picked back up where it
+        assertBusy(() -> {
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
+            assertThat(lifecycleState.getStep(), equalTo(ObservableClusterStateWaitStep.NAME));
+        });
+
+        logger.info("new master is operation");
+        // complete the step
+        AcknowledgedResponse repsonse = client().admin().indices().prepareUpdateSettings("test")
+            .setSettings(Collections.singletonMap("index.lifecycle.test.complete", true)).get();
+
+        assertBusy(() -> {
+            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
+                .prepareState().execute().actionGet().getState().getMetaData().index("test"));
+            assertThat(lifecycleState.getStep(), equalTo(TerminalPolicyStep.KEY.getName()));
+        });
+    }
+
+    public void testPollIntervalUpdate() throws Exception {
+        TimeValue pollInterval = TimeValue.timeValueSeconds(randomLongBetween(1, 5));
+        final String server_1 = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, pollInterval.getStringRep()).build());
+        IndexLifecycleService indexLifecycleService = internalCluster().getInstance(IndexLifecycleService.class, server_1);
+        assertBusy(() -> {
+            assertNotNull(indexLifecycleService.getScheduler());
+            assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
+        });
+        {
+            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule();
+            assertThat(schedule.getInterval(), equalTo(pollInterval));
+        }
+
+        // update the poll interval
+        TimeValue newPollInterval = TimeValue.timeValueHours(randomLongBetween(6, 1000));
+        Settings newIntervalSettings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL,
+            newPollInterval.getStringRep()).build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(newIntervalSettings));
+        {
+            TimeValueSchedule schedule = (TimeValueSchedule) indexLifecycleService.getScheduledJob().getSchedule();
+            assertThat(schedule.getInterval(), equalTo(newPollInterval));
+        }
+    }
+
+    private String getLocalNodeId(String name) {
+        TransportService transportService = internalCluster().getInstance(TransportService.class, name);
+        String nodeId = transportService.getLocalNode().getId();
+        assertThat(nodeId, not(nullValue()));
+        return nodeId;
+    }
+
+    public static class TestILMPlugin extends Plugin {
+        public TestILMPlugin() {
+        }
+
+        public List<Setting<?>> getSettings() {
+            final Setting<Boolean> COMPLETE_SETTING = Setting.boolSetting("index.lifecycle.test.complete", false,
+                Setting.Property.Dynamic, Setting.Property.IndexScope);
+            return Collections.singletonList(COMPLETE_SETTING);
+        }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContent() {
+            return Arrays.asList(
+                new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ObservableAction.NAME), (p) -> {
+                    MockAction.parse(p);
+                    return OBSERVABLE_ACTION;
+                })
+            );
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return Arrays.asList(new NamedWriteableRegistry.Entry(LifecycleType.class, LockableLifecycleType.TYPE,
+                    (in) -> LockableLifecycleType.INSTANCE),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ObservableAction.NAME, ObservableAction::readObservableAction),
+                new NamedWriteableRegistry.Entry(ObservableClusterStateWaitStep.class, ObservableClusterStateWaitStep.NAME,
+                    ObservableClusterStateWaitStep::new));
+        }
+    }
+
+    public static class ObservableClusterStateWaitStep extends ClusterStateWaitStep implements NamedWriteable {
+        public static final String NAME = "observable_cluster_state_action";
+
+        public ObservableClusterStateWaitStep(StepKey current, StepKey next) {
+            super(current, next);
+        }
+
+        public ObservableClusterStateWaitStep(StreamInput in) throws IOException {
+            this(new StepKey(in.readString(), in.readString(), in.readString()), readOptionalNextStepKey(in));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(getKey().getPhase());
+            out.writeString(getKey().getAction());
+            out.writeString(getKey().getName());
+            boolean hasNextStep = getNextStepKey() != null;
+            out.writeBoolean(hasNextStep);
+            if (hasNextStep) {
+                out.writeString(getNextStepKey().getPhase());
+                out.writeString(getNextStepKey().getAction());
+                out.writeString(getNextStepKey().getName());
+            }
+        }
+
+        private static StepKey readOptionalNextStepKey(StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                return new StepKey(in.readString(), in.readString(), in.readString());
+            }
+            return null;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public Result isConditionMet(Index index, ClusterState clusterState) {
+            boolean complete = clusterState.metaData().index("test").getSettings()
+                .getAsBoolean("index.lifecycle.test.complete", false);
+            return new Result(complete, null);
+        }
+    }
+
+    public static class ObservableAction extends MockAction {
+
+        ObservableAction(List<Step> steps, boolean safe) {
+            super(steps, safe);
+        }
+
+        public static ObservableAction readObservableAction(StreamInput in) throws IOException {
+            List<Step> steps = in.readList(ObservableClusterStateWaitStep::new);
+            boolean safe = in.readBoolean();
+            return new ObservableAction(steps, safe);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeList(getSteps().stream().map(s -> (ObservableClusterStateWaitStep) s).collect(Collectors.toList()));
+            out.writeBoolean(isSafeAction());
+        }
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java
new file mode 100644
index 0000000000000..fe24b555a7b27
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MetaData.Custom;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
+import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata.IndexLifecycleMetadataDiff;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.randomTimeseriesLifecyclePolicy;
+
+public class IndexLifecycleMetadataTests extends AbstractDiffableSerializationTestCase<MetaData.Custom> {
+
+    @Override
+    protected IndexLifecycleMetadata createTestInstance() {
+        int numPolicies = randomIntBetween(1, 5);
+        Map<String, LifecyclePolicyMetadata> policies = new HashMap<>(numPolicies);
+        for (int i = 0; i < numPolicies; i++) {
+            LifecyclePolicy policy = randomTimeseriesLifecyclePolicy(randomAlphaOfLength(4) + i);
+            policies.put(policy.getName(), new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong()));
+        }
+        return new IndexLifecycleMetadata(policies, randomFrom(OperationMode.values()));
+    }
+
+    @Override
+    protected IndexLifecycleMetadata doParseInstance(XContentParser parser) throws IOException {
+        return IndexLifecycleMetadata.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected Reader<MetaData.Custom> instanceReader() {
+        return IndexLifecycleMetadata::new;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            Arrays.asList(
+                new NamedWriteableRegistry.Entry(LifecycleType.class, TimeseriesLifecycleType.TYPE,
+                    (in) -> TimeseriesLifecycleType.INSTANCE),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, AllocateAction.NAME, AllocateAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, DeleteAction.NAME, DeleteAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ForceMergeAction.NAME, ForceMergeAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ReadOnlyAction.NAME, ReadOnlyAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, RolloverAction.NAME, RolloverAction::new),
+                new NamedWriteableRegistry.Entry(LifecycleAction.class, ShrinkAction.NAME, ShrinkAction::new)
+            ));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
+        entries.addAll(Arrays.asList(
+            new NamedXContentRegistry.Entry(LifecycleType.class, new ParseField(TimeseriesLifecycleType.TYPE),
+                (p) -> TimeseriesLifecycleType.INSTANCE),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(AllocateAction.NAME), AllocateAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ReadOnlyAction.NAME), ReadOnlyAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
+            new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse)
+        ));
+        return new NamedXContentRegistry(entries);
+    }
+
+    @Override
+    protected MetaData.Custom mutateInstance(MetaData.Custom instance) {
+        IndexLifecycleMetadata metadata = (IndexLifecycleMetadata) instance;
+        Map<String, LifecyclePolicyMetadata> policies = metadata.getPolicyMetadatas();
+        policies = new TreeMap<>(policies);
+        OperationMode mode = metadata.getOperationMode();
+        if (randomBoolean()) {
+            String policyName = randomAlphaOfLength(10);
+            policies.put(policyName, new LifecyclePolicyMetadata(randomTimeseriesLifecyclePolicy(policyName), Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong()));
+        } else {
+            mode = randomValueOtherThan(metadata.getOperationMode(), () -> randomFrom(OperationMode.values()));
+        }
+        return new IndexLifecycleMetadata(policies, mode);
+    }
+
+    @Override
+    protected Custom makeTestChanges(Custom testInstance) {
+        return mutateInstance(testInstance);
+    }
+
+    @Override
+    protected Reader<Diff<Custom>> diffReader() {
+        return IndexLifecycleMetadataDiff::new;
+    }
+
+    public void testMinimumSupportedVersion() {
+        assertEquals(Version.V_6_5_0, createTestInstance().getMinimalSupportedVersion());
+    }
+
+    public void testcontext() {
+        assertEquals(MetaData.ALL_CONTEXTS, createTestInstance().context());
+    }
+
+    public static IndexLifecycleMetadata createTestInstance(int numPolicies, OperationMode mode) {
+        SortedMap<String, LifecyclePolicyMetadata> policies = new TreeMap<>();
+        for (int i = 0; i < numPolicies; i++) {
+            int numberPhases = randomInt(5);
+            Map<String, Phase> phases = new HashMap<>(numberPhases);
+            for (int j = 0; j < numberPhases; j++) {
+                TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+                Map<String, LifecycleAction> actions = Collections.emptyMap();
+                if (randomBoolean()) {
+                    actions = Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
+                }
+                String phaseName = randomAlphaOfLength(10);
+                phases.put(phaseName, new Phase(phaseName, after, actions));
+            }
+            String policyName = randomAlphaOfLength(10);
+            policies.put(policyName, new LifecyclePolicyMetadata(newTestLifecyclePolicy(policyName, phases), Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong()));
+        }
+        return new IndexLifecycleMetadata(policies, mode);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java
new file mode 100644
index 0000000000000..a8a896b5c4ea5
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleRunnerTests.java
@@ -0,0 +1,1479 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.Settings.Builder;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep;
+import org.elasticsearch.xpack.core.indexlifecycle.AsyncWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateActionStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ClusterStateWaitStep;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.MockStep;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+
+public class IndexLifecycleRunnerTests extends ESTestCase {
+    private static final NamedXContentRegistry REGISTRY;
+
+    static {
+        try (IndexLifecycle indexLifecycle = new IndexLifecycle(Settings.EMPTY)) {
+            List<NamedXContentRegistry.Entry> entries = new ArrayList<>(indexLifecycle.getNamedXContent());
+            REGISTRY = new NamedXContentRegistry(entries);
+        }
+    }
+
+    /** A real policy steps registry where getStep can be overridden so that JSON doesn't have to be parsed */
+    private class MockPolicyStepsRegistry extends PolicyStepsRegistry {
+        private BiFunction<IndexMetaData, StepKey, Step> fn = null;
+
+        MockPolicyStepsRegistry(SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap, Map<String, Step> firstStepMap,
+                                Map<String, Map<StepKey, Step>> stepMap, NamedXContentRegistry xContentRegistry, Client client) {
+            super(lifecyclePolicyMap, firstStepMap, stepMap, xContentRegistry, client);
+        }
+
+        public void setResolver(BiFunction<IndexMetaData, StepKey, Step> fn) {
+            this.fn = fn;
+        }
+
+        @Override
+        public Step getStep(IndexMetaData indexMetaData, StepKey stepKey) {
+            if (fn == null) {
+                logger.info("--> retrieving step {}", stepKey);
+                return super.getStep(indexMetaData, stepKey);
+            } else {
+                logger.info("--> returning mock step");
+                return fn.apply(indexMetaData, stepKey);
+            }
+        }
+    }
+
+    private MockPolicyStepsRegistry createOneStepPolicyStepRegistry(String policyName, Step step) {
+        return createOneStepPolicyStepRegistry(policyName, step, "test");
+    }
+
+    private MockPolicyStepsRegistry createOneStepPolicyStepRegistry(String policyName, Step step, String indexName) {
+        LifecyclePolicy policy = new LifecyclePolicy(policyName, new HashMap<>());
+        SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap = new TreeMap<>();
+        lifecyclePolicyMap.put(policyName, new LifecyclePolicyMetadata(policy, new HashMap<>(), 1, 1));
+        Map<String, Step> firstStepMap = new HashMap<>();
+        firstStepMap.put(policyName, step);
+        Map<String, Map<StepKey, Step>> stepMap = new HashMap<>();
+        Map<StepKey, Step> policySteps = new HashMap<>();
+        policySteps.put(step.getKey(), step);
+        stepMap.put(policyName, policySteps);
+        Map<Index, List<Step>> indexSteps = new HashMap<>();
+        List<Step> steps = new ArrayList<>();
+        steps.add(step);
+        Index index = new Index(indexName, indexName + "uuid");
+        indexSteps.put(index, steps);
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        return new MockPolicyStepsRegistry(lifecyclePolicyMap, firstStepMap, stepMap, REGISTRY, client);
+    }
+
+    public void testRunPolicyTerminalPolicyStep() {
+        String policyName = "async_action_policy";
+        TerminalPolicyStep step = TerminalPolicyStep.INSTANCE;
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        Mockito.verifyZeroInteractions(clusterService);
+    }
+
+    public void testRunPolicyErrorStep() {
+        String policyName = "async_action_policy";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        MockClusterStateWaitStep step = new MockClusterStateWaitStep(stepKey, null);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        LifecycleExecutionState.Builder newState = LifecycleExecutionState.builder();
+        newState.setPhase(stepKey.getPhase());
+        newState.setAction(stepKey.getAction());
+        newState.setStep(ErrorStep.NAME);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, newState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        Mockito.verifyZeroInteractions(clusterService);
+    }
+
+    public void testRunStateChangePolicyWithNoNextStep() throws Exception {
+        String policyName = "foo";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        MockClusterStateActionStep step = new MockClusterStateActionStep(stepKey, null);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ThreadPool threadPool = new TestThreadPool("name");
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .build();
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        DiscoveryNode node = clusterService.localNode();
+        IndexLifecycleMetadata ilm = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .metaData(MetaData.builder()
+                .put(indexMetaData, true)
+                .putCustom(IndexLifecycleMetadata.TYPE, ilm))
+            .nodes(DiscoveryNodes.builder()
+                .add(node)
+                .masterNodeId(node.getId())
+                .localNodeId(node.getId()))
+            .build();
+        ClusterServiceUtils.setState(clusterService, state);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+
+        ClusterState before = clusterService.state();
+        CountDownLatch latch = new CountDownLatch(1);
+        step.setLatch(latch);
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        latch.await(5, TimeUnit.SECONDS);
+        ClusterState after = clusterService.state();
+
+        assertEquals(before, after);
+        assertThat(step.getExecuteCount(), equalTo(1L));
+        clusterService.close();
+        threadPool.shutdownNow();
+    }
+
+    public void testRunStateChangePolicyWithNextStep() throws Exception {
+        String policyName = "foo";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        StepKey nextStepKey = new StepKey("phase", "action", "next_cluster_state_action_step");
+        MockClusterStateActionStep step = new MockClusterStateActionStep(stepKey, nextStepKey);
+        MockClusterStateActionStep nextStep = new MockClusterStateActionStep(nextStepKey, null);
+        MockPolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        stepRegistry.setResolver((i, k) -> {
+            if (stepKey.equals(k)) {
+                return step;
+            } else if (nextStepKey.equals(k)) {
+                return nextStep;
+            } else {
+                fail("should not try to retrieve different step");
+                return null;
+            }
+        });
+        ThreadPool threadPool = new TestThreadPool("name");
+        LifecycleExecutionState les = LifecycleExecutionState.builder()
+            .setPhase("phase")
+            .setAction("action")
+            .setStep("cluster_state_action_step")
+            .build();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, les.asMap())
+            .build();
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        DiscoveryNode node = clusterService.localNode();
+        IndexLifecycleMetadata ilm = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .metaData(MetaData.builder()
+                .put(indexMetaData, true)
+                .putCustom(IndexLifecycleMetadata.TYPE, ilm))
+            .nodes(DiscoveryNodes.builder()
+                .add(node)
+                .masterNodeId(node.getId())
+                .localNodeId(node.getId()))
+            .build();
+        ClusterServiceUtils.setState(clusterService, state);
+        long stepTime = randomLong();
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> stepTime);
+
+        ClusterState before = clusterService.state();
+        CountDownLatch latch = new CountDownLatch(1);
+        nextStep.setLatch(latch);
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+        // The cluster state can take a few extra milliseconds to update after the steps are executed
+        assertBusy(() -> assertNotEquals(before, clusterService.state()));
+        LifecycleExecutionState newExecutionState = LifecycleExecutionState
+            .fromIndexMetadata(clusterService.state().metaData().index(indexMetaData.getIndex()));
+        assertThat(newExecutionState.getPhase(), equalTo("phase"));
+        assertThat(newExecutionState.getAction(), equalTo("action"));
+        assertThat(newExecutionState.getStep(), equalTo("next_cluster_state_action_step"));
+        assertThat(newExecutionState.getStepTime(), equalTo(stepTime));
+        assertThat(step.getExecuteCount(), equalTo(1L));
+        assertThat(nextStep.getExecuteCount(), equalTo(1L));
+        clusterService.close();
+        threadPool.shutdownNow();
+    }
+
+    public void testRunAsyncActionDoesNotRun() {
+        String policyName = "foo";
+        StepKey stepKey = new StepKey("phase", "action", "async_action_step");
+        MockAsyncActionStep step = new MockAsyncActionStep(stepKey, null);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ThreadPool threadPool = new TestThreadPool("name");
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .build();
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        DiscoveryNode node = clusterService.localNode();
+        IndexLifecycleMetadata ilm = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .metaData(MetaData.builder()
+                .put(indexMetaData, true)
+                .putCustom(IndexLifecycleMetadata.TYPE, ilm))
+            .nodes(DiscoveryNodes.builder()
+                .add(node)
+                .masterNodeId(node.getId())
+                .localNodeId(node.getId()))
+            .build();
+        ClusterServiceUtils.setState(clusterService, state);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+
+        ClusterState before = clusterService.state();
+        // State changes should not run AsyncAction steps
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        ClusterState after = clusterService.state();
+
+        assertEquals(before, after);
+        assertThat(step.getExecuteCount(), equalTo(0L));
+        clusterService.close();
+        threadPool.shutdownNow();
+    }
+
+    public void testRunStateChangePolicyWithAsyncActionNextStep() throws Exception {
+        String policyName = "foo";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        StepKey nextStepKey = new StepKey("phase", "action", "async_action_step");
+        MockClusterStateActionStep step = new MockClusterStateActionStep(stepKey, nextStepKey);
+        MockAsyncActionStep nextStep = new MockAsyncActionStep(nextStepKey, null);
+        MockPolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        stepRegistry.setResolver((i, k) -> {
+            if (stepKey.equals(k)) {
+                return step;
+            } else if (nextStepKey.equals(k)) {
+                return nextStep;
+            } else {
+                fail("should not try to retrieve different step");
+                return null;
+            }
+        });
+        ThreadPool threadPool = new TestThreadPool("name");
+        LifecycleExecutionState les = LifecycleExecutionState.builder()
+            .setPhase("phase")
+            .setAction("action")
+            .setStep("cluster_state_action_step")
+            .build();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, les.asMap())
+            .build();
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        DiscoveryNode node = clusterService.localNode();
+        IndexLifecycleMetadata ilm = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .metaData(MetaData.builder()
+                .put(indexMetaData, true)
+                .putCustom(IndexLifecycleMetadata.TYPE, ilm))
+            .nodes(DiscoveryNodes.builder()
+                .add(node)
+                .masterNodeId(node.getId())
+                .localNodeId(node.getId()))
+            .build();
+        logger.info("--> state: {}", state);
+        ClusterServiceUtils.setState(clusterService, state);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+
+        ClusterState before = clusterService.state();
+        CountDownLatch latch = new CountDownLatch(1);
+        step.setLatch(latch);
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        // Wait for the cluster state action step
+        latch.await(5, TimeUnit.SECONDS);
+
+        CountDownLatch asyncLatch = new CountDownLatch(1);
+        nextStep.setLatch(asyncLatch);
+
+        // Wait for the async action step
+        asyncLatch.await(5, TimeUnit.SECONDS);
+        ClusterState after = clusterService.state();
+
+        assertNotEquals(before, after);
+        assertThat(step.getExecuteCount(), equalTo(1L));
+        assertThat(nextStep.getExecuteCount(), equalTo(1L));
+        clusterService.close();
+        threadPool.shutdownNow();
+    }
+
+    public void testRunPeriodicStep() throws Exception {
+        String policyName = "foo";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        StepKey nextStepKey = new StepKey("phase", "action", "async_action_step");
+        MockAsyncWaitStep step = new MockAsyncWaitStep(stepKey, nextStepKey);
+        MockAsyncWaitStep nextStep = new MockAsyncWaitStep(nextStepKey, null);
+        MockPolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        stepRegistry.setResolver((i, k) -> {
+            if (stepKey.equals(k)) {
+                return step;
+            } else if (nextStepKey.equals(k)) {
+                return nextStep;
+            } else {
+                fail("should not try to retrieve different step");
+                return null;
+            }
+        });
+        ThreadPool threadPool = new TestThreadPool("name");
+        LifecycleExecutionState les = LifecycleExecutionState.builder()
+            .setPhase("phase")
+            .setAction("action")
+            .setStep("cluster_state_action_step")
+            .build();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+            .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, les.asMap())
+            .build();
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        DiscoveryNode node = clusterService.localNode();
+        IndexLifecycleMetadata ilm = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        ClusterState state = ClusterState.builder(new ClusterName("cluster"))
+            .metaData(MetaData.builder()
+                .put(indexMetaData, true)
+                .putCustom(IndexLifecycleMetadata.TYPE, ilm))
+            .nodes(DiscoveryNodes.builder()
+                .add(node)
+                .masterNodeId(node.getId())
+                .localNodeId(node.getId()))
+            .build();
+        logger.info("--> state: {}", state);
+        ClusterServiceUtils.setState(clusterService, state);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+
+        ClusterState before = clusterService.state();
+        CountDownLatch latch = new CountDownLatch(1);
+        step.setLatch(latch);
+        runner.runPeriodicStep(policyName, indexMetaData);
+        latch.await(5, TimeUnit.SECONDS);
+
+        ClusterState after = clusterService.state();
+
+        assertEquals(before, after);
+        assertThat(step.getExecuteCount(), equalTo(1L));
+        assertThat(nextStep.getExecuteCount(), equalTo(0L));
+        clusterService.close();
+        threadPool.shutdownNow();
+    }
+
+    public void testRunPolicyClusterStateActionStep() {
+        String policyName = "cluster_state_action_policy";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        MockClusterStateActionStep step = new MockClusterStateActionStep(stepKey, null);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        Mockito.verify(clusterService, Mockito.times(1)).submitStateUpdateTask(Mockito.matches("ilm-execute-cluster-state-steps"),
+                Mockito.argThat(new ExecuteStepsUpdateTaskMatcher(indexMetaData.getIndex(), policyName, step)));
+        Mockito.verifyNoMoreInteractions(clusterService);
+    }
+
+    public void testRunPolicyClusterStateWaitStep() {
+        String policyName = "cluster_state_action_policy";
+        StepKey stepKey = new StepKey("phase", "action", "cluster_state_action_step");
+        MockClusterStateWaitStep step = new MockClusterStateWaitStep(stepKey, null);
+        step.setWillComplete(true);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        Mockito.verify(clusterService, Mockito.times(1)).submitStateUpdateTask(Mockito.matches("ilm-execute-cluster-state-steps"),
+                Mockito.argThat(new ExecuteStepsUpdateTaskMatcher(indexMetaData.getIndex(), policyName, step)));
+        Mockito.verifyNoMoreInteractions(clusterService);
+    }
+
+    public void testRunPolicyAsyncActionStepClusterStateChangeIgnored() {
+        String policyName = "async_action_policy";
+        StepKey stepKey = new StepKey("phase", "action", "async_action_step");
+        MockAsyncActionStep step = new MockAsyncActionStep(stepKey, null);
+        Exception expectedException = new RuntimeException();
+        step.setException(expectedException);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        assertEquals(0, step.getExecuteCount());
+        Mockito.verifyZeroInteractions(clusterService);
+    }
+
+    public void testRunPolicyAsyncWaitStepClusterStateChangeIgnored() {
+        String policyName = "async_wait_policy";
+        StepKey stepKey = new StepKey("phase", "action", "async_wait_step");
+        MockAsyncWaitStep step = new MockAsyncWaitStep(stepKey, null);
+        Exception expectedException = new RuntimeException();
+        step.setException(expectedException);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(stepRegistry, clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+
+        assertEquals(0, step.getExecuteCount());
+        Mockito.verifyZeroInteractions(clusterService);
+    }
+
+    public void testRunPolicyThatDoesntExist() {
+        String policyName = "cluster_state_action_policy";
+        ClusterService clusterService = mock(ClusterService.class);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(new PolicyStepsRegistry(NamedXContentRegistry.EMPTY, null),
+            clusterService, () -> 0L);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        // verify that no exception is thrown
+        runner.runPolicyAfterStateChange(policyName, indexMetaData);
+        Mockito.verify(clusterService, Mockito.times(1)).submitStateUpdateTask(Mockito.matches("ilm-set-step-info"),
+            Mockito.argThat(new SetStepInfoUpdateTaskMatcher(indexMetaData.getIndex(), policyName, null,
+                (builder, params) -> {
+                    builder.startObject();
+                    builder.field("reason", "policy [does_not_exist] does not exist");
+                    builder.field("type", "illegal_argument_exception");
+                    builder.endObject();
+                    return builder;
+                })));
+        Mockito.verifyNoMoreInteractions(clusterService);
+    }
+
+    public void testGetCurrentStepKey() {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState.build());
+        assertNull(stepKey);
+
+        String phase = randomAlphaOfLength(20);
+        String action = randomAlphaOfLength(20);
+        String step = randomAlphaOfLength(20);
+        LifecycleExecutionState.Builder lifecycleState2 = LifecycleExecutionState.builder();
+        lifecycleState2.setPhase(phase);
+        lifecycleState2.setAction(action);
+        lifecycleState2.setStep(step);
+        stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState2.build());
+        assertNotNull(stepKey);
+        assertEquals(phase, stepKey.getPhase());
+        assertEquals(action, stepKey.getAction());
+        assertEquals(step, stepKey.getName());
+
+        phase = randomAlphaOfLength(20);
+        action = randomAlphaOfLength(20);
+        step = null;
+        LifecycleExecutionState.Builder lifecycleState3 = LifecycleExecutionState.builder();
+        lifecycleState3.setPhase(phase);
+        lifecycleState3.setAction(action);
+        lifecycleState3.setStep(step);
+        AssertionError error3 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState3.build()));
+        assertEquals("Current phase is not empty: " + phase, error3.getMessage());
+
+        phase = null;
+        action = randomAlphaOfLength(20);
+        step = null;
+        LifecycleExecutionState.Builder lifecycleState4 = LifecycleExecutionState.builder();
+        lifecycleState4.setPhase(phase);
+        lifecycleState4.setAction(action);
+        lifecycleState4.setStep(step);
+        AssertionError error4 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState4.build()));
+        assertEquals("Current action is not empty: " + action, error4.getMessage());
+
+        phase = null;
+        action = randomAlphaOfLength(20);
+        step = randomAlphaOfLength(20);
+        LifecycleExecutionState.Builder lifecycleState5 = LifecycleExecutionState.builder();
+        lifecycleState5.setPhase(phase);
+        lifecycleState5.setAction(action);
+        lifecycleState5.setStep(step);
+        AssertionError error5 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState5.build()));
+        assertEquals(null, error5.getMessage());
+
+        phase = null;
+        action = null;
+        step = randomAlphaOfLength(20);
+        LifecycleExecutionState.Builder lifecycleState6 = LifecycleExecutionState.builder();
+        lifecycleState6.setPhase(phase);
+        lifecycleState6.setAction(action);
+        lifecycleState6.setStep(step);
+        AssertionError error6 = expectThrows(AssertionError.class, () -> IndexLifecycleRunner.getCurrentStepKey(lifecycleState6.build()));
+        assertEquals(null, error6.getMessage());
+    }
+
+    public void testGetCurrentStep() {
+        String policyName = "policy";
+        StepKey firstStepKey = new StepKey("phase_1", "action_1", "step_1");
+        StepKey secondStepKey = new StepKey("phase_1", "action_1", "step_2");
+        Step firstStep = new MockStep(firstStepKey, secondStepKey);
+        Map<String, Step> firstStepMap = new HashMap<>();
+        firstStepMap.put(policyName, firstStep);
+        Map<String, Map<StepKey, Step>> stepMap = new HashMap<>();
+        Index index = new Index("test", "uuid");
+
+        Step.StepKey MOCK_STEP_KEY = new Step.StepKey("mock", "mock", "mock");
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        LifecyclePolicy policy = LifecyclePolicyTests.randomTimeseriesLifecyclePolicyWithAllPhases(policyName);
+        LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), 1, randomNonNegativeLong());
+        String phaseName = randomFrom(policy.getPhases().keySet());
+        Phase phase = policy.getPhases().get(phaseName);
+        PhaseExecutionInfo pei = new PhaseExecutionInfo(policy.getName(), phase, 1, randomNonNegativeLong());
+        String phaseJson = Strings.toString(pei);
+        LifecycleAction action = randomFrom(phase.getActions().values());
+        Step step = randomFrom(action.toSteps(client, phaseName, MOCK_STEP_KEY));
+        Settings indexSettings = Settings.builder()
+            .put("index.number_of_shards", 1)
+            .put("index.number_of_replicas", 0)
+            .put("index.version.created", Version.CURRENT)
+            .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
+            .build();
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
+        lifecycleState.setPhase(step.getKey().getPhase());
+        lifecycleState.setAction(step.getKey().getAction());
+        lifecycleState.setStep(step.getKey().getName());
+        IndexMetaData indexMetaData = IndexMetaData.builder(index.getName())
+            .settings(indexSettings)
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
+        SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
+        metas.put(policyName, policyMetadata);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(metas, firstStepMap, stepMap, REGISTRY, client);
+
+        // First step is retrieved because there are no settings for the index
+        Step stepFromNoSettings = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData,
+            LifecycleExecutionState.builder().build());
+        assertEquals(firstStep, stepFromNoSettings);
+
+        // The step that was written into the metadata is retrieved
+        Step currentStep = IndexLifecycleRunner.getCurrentStep(registry, policy.getName(), indexMetaData, lifecycleState.build());
+        assertEquals(step.getKey(), currentStep.getKey());
+    }
+
+    public void testMoveClusterStateToNextStep() {
+        String indexName = "my_index";
+        LifecyclePolicy policy = randomValueOtherThanMany(p -> p.getPhases().size() == 0,
+            () -> LifecyclePolicyTests.randomTestLifecyclePolicy("policy"));
+        Phase nextPhase = policy.getPhases().values().stream().findFirst().get();
+        List<LifecyclePolicyMetadata> policyMetadatas = Collections.singletonList(
+            new LifecyclePolicyMetadata(policy, Collections.emptyMap(), randomNonNegativeLong(), randomNonNegativeLong()));
+        StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStep = new StepKey(nextPhase.getName(), "next_action", "next_step");
+        long now = randomNonNegativeLong();
+
+        // test going from null lifecycle settings to next step
+        ClusterState clusterState = buildClusterState(indexName,
+            Settings.builder()
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName()), LifecycleExecutionState.builder().build(), policyMetadatas);
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
+                () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        // test going from set currentStep settings to nextStep
+        Builder indexSettingsBuilder = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_NAME, policy.getName());
+        if (randomBoolean()) {
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
+        }
+
+        clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
+        index = clusterState.metaData().index(indexName).getIndex();
+        newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+    }
+
+    public void testMoveClusterStateToNextStepSamePhase() {
+        String indexName = "my_index";
+        StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStep = new StepKey("current_phase", "next_action", "next_step");
+        long now = randomNonNegativeLong();
+
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
+                () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        if (randomBoolean()) {
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
+        }
+
+        clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
+        index = clusterState.metaData().index(indexName).getIndex();
+        newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+    }
+
+    public void testMoveClusterStateToNextStepSameAction() {
+        String indexName = "my_index";
+        StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStep = new StepKey("current_phase", "current_action", "next_step");
+        long now = randomNonNegativeLong();
+
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep,
+                () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        if (randomBoolean()) {
+            lifecycleState.setStepInfo(randomAlphaOfLength(20));
+        }
+        clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
+        index = clusterState.metaData().index(indexName).getIndex();
+        newClusterState = IndexLifecycleRunner.moveClusterStateToNextStep(index, clusterState, currentStep, nextStep, () -> now);
+        assertClusterStateOnNextStep(clusterState, index, currentStep, nextStep, newClusterState, now);
+    }
+
+    public void testSuccessfulValidatedMoveClusterStateToNextStep() {
+        String indexName = "my_index";
+        String policyName = "my_policy";
+        LifecyclePolicy policy = randomValueOtherThanMany(p -> p.getPhases().size() == 0,
+            () -> LifecyclePolicyTests.randomTestLifecyclePolicy(policyName));
+        Phase nextPhase = policy.getPhases().values().stream().findFirst().get();
+        List<LifecyclePolicyMetadata> policyMetadatas = Collections.singletonList(
+            new LifecyclePolicyMetadata(policy, Collections.emptyMap(), randomNonNegativeLong(), randomNonNegativeLong()));
+        StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStepKey = new StepKey(nextPhase.getName(), "next_action", "next_step");
+        long now = randomNonNegativeLong();
+        Step step = new MockStep(nextStepKey, nextStepKey);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName);
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
+            nextStepKey, () -> now, stepRegistry);
+        assertClusterStateOnNextStep(clusterState, index, currentStepKey, nextStepKey, newClusterState, now);
+    }
+
+    public void testValidatedMoveClusterStateToNextStepWithoutPolicy() {
+        String indexName = "my_index";
+        String policyName = "policy";
+        StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStepKey = new StepKey("next_phase", "next_action", "next_step");
+        long now = randomNonNegativeLong();
+        Step step = new MockStep(nextStepKey, nextStepKey);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, randomBoolean() ? "" : null);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
+                nextStepKey, () -> now, stepRegistry));
+        assertThat(exception.getMessage(), equalTo("index [my_index] is not associated with an Index Lifecycle Policy"));
+    }
+
+    public void testValidatedMoveClusterStateToNextStepInvalidCurrentStep() {
+        String indexName = "my_index";
+        String policyName = "my_policy";
+        StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey notCurrentStepKey = new StepKey("not_current_phase", "not_current_action", "not_current_step");
+        StepKey nextStepKey = new StepKey("next_phase", "next_action", "next_step");
+        long now = randomNonNegativeLong();
+        Step step = new MockStep(nextStepKey, nextStepKey);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, notCurrentStepKey,
+                nextStepKey, () -> now, stepRegistry));
+        assertThat(exception.getMessage(), equalTo("index [my_index] is not on current step " +
+            "[{\"phase\":\"not_current_phase\",\"action\":\"not_current_action\",\"name\":\"not_current_step\"}]"));
+    }
+
+    public void testValidatedMoveClusterStateToNextStepInvalidNextStep() {
+        String indexName = "my_index";
+        String policyName = "my_policy";
+        StepKey currentStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey nextStepKey = new StepKey("next_phase", "next_action", "next_step");
+        long now = randomNonNegativeLong();
+        Step step = new MockStep(currentStepKey, nextStepKey);
+        PolicyStepsRegistry stepRegistry = createOneStepPolicyStepRegistry(policyName, step);
+
+        Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
+
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> IndexLifecycleRunner.moveClusterStateToStep(indexName, clusterState, currentStepKey,
+                nextStepKey, () -> now, stepRegistry));
+        assertThat(exception.getMessage(),
+            equalTo("step [{\"phase\":\"next_phase\",\"action\":\"next_action\",\"name\":\"next_step\"}] " +
+                "for index [my_index] with policy [my_policy] does not exist"));
+    }
+
+    public void testMoveClusterStateToErrorStep() throws IOException {
+        String indexName = "my_index";
+        StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
+        long now = randomNonNegativeLong();
+        Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+
+        ClusterState newClusterState = IndexLifecycleRunner.moveClusterStateToErrorStep(index, clusterState, currentStep, cause, () -> now);
+        assertClusterStateOnErrorStep(clusterState, index, currentStep, newClusterState, now,
+            "{\"type\":\"exception\",\"reason\":\"THIS IS AN EXPECTED CAUSE\"}");
+
+        cause = new IllegalArgumentException("non elasticsearch-exception");
+        newClusterState = IndexLifecycleRunner.moveClusterStateToErrorStep(index, clusterState, currentStep, cause, () -> now);
+        assertClusterStateOnErrorStep(clusterState, index, currentStep, newClusterState, now,
+            "{\"type\":\"illegal_argument_exception\",\"reason\":\"non elasticsearch-exception\"}");
+    }
+
+    public void testMoveClusterStateToFailedStep() {
+        String indexName = "my_index";
+        String[] indices = new String[] { indexName };
+        String policyName = "my_policy";
+        long now = randomNonNegativeLong();
+        StepKey failedStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey errorStepKey = new StepKey(failedStepKey.getPhase(), failedStepKey.getAction(), ErrorStep.NAME);
+        Step step = new MockStep(failedStepKey, null);
+        PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step, indexName);
+        Settings.Builder indexSettingsBuilder = Settings.builder()
+                .put(LifecycleSettings.LIFECYCLE_NAME, policyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(errorStepKey.getPhase());
+        lifecycleState.setAction(errorStepKey.getAction());
+        lifecycleState.setStep(errorStepKey.getName());
+        lifecycleState.setFailedStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
+        ClusterState nextClusterState = runner.moveClusterStateToFailedStep(clusterState, indices);
+        IndexLifecycleRunnerTests.assertClusterStateOnNextStep(clusterState, index, errorStepKey, failedStepKey,
+            nextClusterState, now);
+    }
+
+    public void testMoveClusterStateToFailedStepIndexNotFound() {
+        String existingIndexName = "my_index";
+        String invalidIndexName = "does_not_exist";
+        ClusterState clusterState = buildClusterState(existingIndexName, Settings.builder(), LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(null, null, () -> 0L);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> runner.moveClusterStateToFailedStep(clusterState, new String[] { invalidIndexName }));
+        assertThat(exception.getMessage(), equalTo("index [" + invalidIndexName + "] does not exist"));
+    }
+
+    public void testMoveClusterStateToFailedStepInvalidPolicySetting() {
+        String indexName = "my_index";
+        String[] indices = new String[] { indexName };
+        String policyName = "my_policy";
+        long now = randomNonNegativeLong();
+        StepKey failedStepKey = new StepKey("current_phase", "current_action", "current_step");
+        StepKey errorStepKey = new StepKey(failedStepKey.getPhase(), failedStepKey.getAction(), ErrorStep.NAME);
+        Step step = new MockStep(failedStepKey, null);
+        PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        Settings.Builder indexSettingsBuilder = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(errorStepKey.getPhase());
+        lifecycleState.setAction(errorStepKey.getAction());
+        lifecycleState.setStep(errorStepKey.getName());
+        lifecycleState.setFailedStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> runner.moveClusterStateToFailedStep(clusterState, indices));
+        assertThat(exception.getMessage(), equalTo("index [" + indexName + "] is not associated with an Index Lifecycle Policy"));
+    }
+
+    public void testMoveClusterStateToFailedNotOnError() {
+        String indexName = "my_index";
+        String[] indices = new String[] { indexName };
+        String policyName = "my_policy";
+        long now = randomNonNegativeLong();
+        StepKey failedStepKey = new StepKey("current_phase", "current_action", "current_step");
+        Step step = new MockStep(failedStepKey, null);
+        PolicyStepsRegistry policyRegistry = createOneStepPolicyStepRegistry(policyName, step);
+        Settings.Builder indexSettingsBuilder = Settings.builder()
+            .put(LifecycleSettings.LIFECYCLE_NAME, (String) null);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(failedStepKey.getPhase());
+        lifecycleState.setAction(failedStepKey.getAction());
+        lifecycleState.setStep(failedStepKey.getName());
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), Collections.emptyList());
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(policyRegistry, null, () -> now);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> runner.moveClusterStateToFailedStep(clusterState, indices));
+        assertThat(exception.getMessage(), equalTo("cannot retry an action for an index [" + indices[0]
+            + "] that has not encountered an error when running a Lifecycle Policy"));
+    }
+
+    public void testAddStepInfoToClusterState() throws IOException {
+        String indexName = "my_index";
+        StepKey currentStep = new StepKey("current_phase", "current_action", "current_step");
+        RandomStepInfo stepInfo = new RandomStepInfo(() -> randomAlphaOfLength(10));
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        ClusterState clusterState = buildClusterState(indexName, Settings.builder(), lifecycleState.build(), Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        ClusterState newClusterState = IndexLifecycleRunner.addStepInfoToClusterState(index, clusterState, stepInfo);
+        assertClusterStateStepInfo(clusterState, index, currentStep, newClusterState, stepInfo);
+        ClusterState runAgainClusterState = IndexLifecycleRunner.addStepInfoToClusterState(index, newClusterState, stepInfo);
+        assertSame(newClusterState, runAgainClusterState);
+    }
+
+    private ClusterState buildClusterState(String indexName, Settings.Builder indexSettingsBuilder,
+                                           LifecycleExecutionState lifecycleState,
+                                           List<LifecyclePolicyMetadata> lifecyclePolicyMetadatas) {
+        Settings indexSettings = indexSettingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+        IndexMetaData indexMetadata = IndexMetaData.builder(indexName)
+            .settings(indexSettings)
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap())
+            .build();
+
+        Map<String, LifecyclePolicyMetadata> lifecyclePolicyMetadatasMap = lifecyclePolicyMetadatas.stream()
+                .collect(Collectors.toMap(LifecyclePolicyMetadata::getName, Function.identity()));
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(lifecyclePolicyMetadatasMap, OperationMode.RUNNING);
+
+        MetaData metadata = MetaData.builder().put(indexMetadata, true).putCustom(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata)
+                .build();
+        return ClusterState.builder(new ClusterName("my_cluster")).metaData(metadata).build();
+    }
+
+    private static LifecyclePolicy createPolicy(String policyName, StepKey safeStep, StepKey unsafeStep) {
+        Map<String, Phase> phases = new HashMap<>();
+        if (safeStep != null) {
+            assert MockAction.NAME.equals(safeStep.getAction()) : "The safe action needs to be MockAction.NAME";
+            assert unsafeStep == null
+                    || safeStep.getPhase().equals(unsafeStep.getPhase()) == false : "safe and unsafe actions must be in different phases";
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            List<Step> steps = Collections.singletonList(new MockStep(safeStep, null));
+            MockAction safeAction = new MockAction(steps, true);
+            actions.put(safeAction.getWriteableName(), safeAction);
+            Phase phase = new Phase(safeStep.getPhase(), TimeValue.timeValueMillis(0), actions);
+            phases.put(phase.getName(), phase);
+        }
+        if (unsafeStep != null) {
+            assert MockAction.NAME.equals(unsafeStep.getAction()) : "The unsafe action needs to be MockAction.NAME";
+            Map<String, LifecycleAction> actions = new HashMap<>();
+            List<Step> steps = Collections.singletonList(new MockStep(unsafeStep, null));
+            MockAction unsafeAction = new MockAction(steps, false);
+            actions.put(unsafeAction.getWriteableName(), unsafeAction);
+            Phase phase = new Phase(unsafeStep.getPhase(), TimeValue.timeValueMillis(0), actions);
+            phases.put(phase.getName(), phase);
+        }
+        return newTestLifecyclePolicy(policyName, phases);
+    }
+
+    public void testRemovePolicyForIndex() {
+        String indexName = randomAlphaOfLength(10);
+        String oldPolicyName = "old_policy";
+        StepKey currentStep = new StepKey(randomAlphaOfLength(10), MockAction.NAME, randomAlphaOfLength(10));
+        LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, currentStep, null);
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
+        policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        Index[] indices = new Index[] { index };
+        List<String> failedIndexes = new ArrayList<>();
+
+        ClusterState newClusterState = IndexLifecycleRunner.removePolicyForIndexes(indices, clusterState, failedIndexes);
+
+        assertTrue(failedIndexes.isEmpty());
+        assertIndexNotManagedByILM(newClusterState, index);
+    }
+
+    public void testRemovePolicyForIndexNoCurrentPolicy() {
+        String indexName = randomAlphaOfLength(10);
+        Settings.Builder indexSettingsBuilder = Settings.builder();
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, LifecycleExecutionState.builder().build(),
+            Collections.emptyList());
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        Index[] indices = new Index[] { index };
+        List<String> failedIndexes = new ArrayList<>();
+
+        ClusterState newClusterState = IndexLifecycleRunner.removePolicyForIndexes(indices, clusterState, failedIndexes);
+
+        assertTrue(failedIndexes.isEmpty());
+        assertIndexNotManagedByILM(newClusterState, index);
+    }
+
+    public void testRemovePolicyForIndexIndexDoesntExist() {
+        String indexName = randomAlphaOfLength(10);
+        String oldPolicyName = "old_policy";
+        LifecyclePolicy oldPolicy = newTestLifecyclePolicy(oldPolicyName, Collections.emptyMap());
+        StepKey currentStep = AbstractStepTestCase.randomStepKey();
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
+        policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
+        Index index = new Index("doesnt_exist", "im_not_here");
+        Index[] indices = new Index[] { index };
+        List<String> failedIndexes = new ArrayList<>();
+
+        ClusterState newClusterState = IndexLifecycleRunner.removePolicyForIndexes(indices, clusterState, failedIndexes);
+
+        assertEquals(1, failedIndexes.size());
+        assertEquals("doesnt_exist", failedIndexes.get(0));
+        assertSame(clusterState, newClusterState);
+    }
+
+    public void testRemovePolicyForIndexIndexInUnsafe() {
+        String indexName = randomAlphaOfLength(10);
+        String oldPolicyName = "old_policy";
+        StepKey currentStep = new StepKey(randomAlphaOfLength(10), MockAction.NAME, randomAlphaOfLength(10));
+        LifecyclePolicy oldPolicy = createPolicy(oldPolicyName, null, currentStep);
+        Settings.Builder indexSettingsBuilder = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, oldPolicyName);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStep.getPhase());
+        lifecycleState.setAction(currentStep.getAction());
+        lifecycleState.setStep(currentStep.getName());
+        List<LifecyclePolicyMetadata> policyMetadatas = new ArrayList<>();
+        policyMetadatas.add(new LifecyclePolicyMetadata(oldPolicy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        ClusterState clusterState = buildClusterState(indexName, indexSettingsBuilder, lifecycleState.build(), policyMetadatas);
+        Index index = clusterState.metaData().index(indexName).getIndex();
+        Index[] indices = new Index[] { index };
+        List<String> failedIndexes = new ArrayList<>();
+
+        ClusterState newClusterState = IndexLifecycleRunner.removePolicyForIndexes(indices, clusterState, failedIndexes);
+
+        assertTrue(failedIndexes.isEmpty());
+        assertIndexNotManagedByILM(newClusterState, index);
+    }
+
+    public void testIsReadyToTransition() {
+        String policyName = "async_action_policy";
+        StepKey stepKey = new StepKey("phase", MockAction.NAME, MockAction.NAME);
+        MockAsyncActionStep step = new MockAsyncActionStep(stepKey, null);
+        step.setWillComplete(true);
+        SortedMap<String, LifecyclePolicyMetadata> lifecyclePolicyMap = new TreeMap<>(Collections.singletonMap(policyName,
+            new LifecyclePolicyMetadata(createPolicy(policyName, null, step.getKey()), new HashMap<>(),
+                randomNonNegativeLong(), randomNonNegativeLong())));
+        Map<String, Step> firstStepMap = Collections.singletonMap(policyName, step);
+        Map<StepKey, Step> policySteps = Collections.singletonMap(step.getKey(), step);
+        Map<String, Map<StepKey, Step>> stepMap = Collections.singletonMap(policyName, policySteps);
+        PolicyStepsRegistry policyStepsRegistry = new PolicyStepsRegistry(lifecyclePolicyMap, firstStepMap,
+            stepMap, NamedXContentRegistry.EMPTY, null);
+        ClusterService clusterService = mock(ClusterService.class);
+        final AtomicLong now = new AtomicLong(5);
+        IndexLifecycleRunner runner = new IndexLifecycleRunner(policyStepsRegistry, clusterService, now::get);
+        IndexMetaData indexMetaData = IndexMetaData.builder("my_index").settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5))
+            .numberOfReplicas(randomIntBetween(0, 5))
+            .build();
+        // With no time, always transition
+        assertTrue("index should be able to transition with no creation date",
+            runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
+
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setIndexCreationDate(10L);
+        indexMetaData = IndexMetaData.builder(indexMetaData)
+            .settings(Settings.builder()
+                .put(indexMetaData.getSettings())
+                .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
+        // Index is not old enough to transition
+        assertFalse("index is not able to transition if it isn't old enough",
+            runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
+
+        // Set to the fuuuuuttuuuuuuurre
+        now.set(Long.MAX_VALUE);
+        assertTrue("index should be able to transition past phase's age",
+            runner.isReadyToTransitionToThisPhase(policyName, indexMetaData, "phase"));
+    }
+
+
+    public static void assertIndexNotManagedByILM(ClusterState clusterState, Index index) {
+        MetaData metadata = clusterState.metaData();
+        assertNotNull(metadata);
+        IndexMetaData indexMetadata = metadata.getIndexSafe(index);
+        assertNotNull(indexMetadata);
+        Settings indexSettings = indexMetadata.getSettings();
+        assertNotNull(indexSettings);
+        assertFalse(LifecycleSettings.LIFECYCLE_NAME_SETTING.exists(indexSettings));
+        assertFalse(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.exists(indexSettings));
+    }
+
+    public static void assertClusterStateOnPolicy(ClusterState oldClusterState, Index index, String expectedPolicy, StepKey previousStep,
+            StepKey expectedStep, ClusterState newClusterState, long now) {
+        assertNotSame(oldClusterState, newClusterState);
+        MetaData newMetadata = newClusterState.metaData();
+        assertNotSame(oldClusterState.metaData(), newMetadata);
+        IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
+        assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(index));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
+        assertNotSame(oldLifecycleState, newLifecycleState);
+        assertEquals(expectedStep.getPhase(), newLifecycleState.getPhase());
+        assertEquals(expectedStep.getAction(), newLifecycleState.getAction());
+        assertEquals(expectedStep.getName(), newLifecycleState.getStep());
+        if (Objects.equals(previousStep.getPhase(), expectedStep.getPhase())) {
+            assertEquals(oldLifecycleState.getPhase(), newLifecycleState.getPhase());
+        } else {
+            assertEquals(now, newLifecycleState.getPhaseTime().longValue());
+        }
+        if (Objects.equals(previousStep.getAction(), expectedStep.getAction())) {
+            assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
+        } else {
+            assertEquals(now, newLifecycleState.getActionTime().longValue());
+        }
+        if (Objects.equals(previousStep.getName(), expectedStep.getName())) {
+            assertEquals(oldLifecycleState.getStepTime(), newLifecycleState.getStepTime());
+        } else {
+            assertEquals(now, newLifecycleState.getStepTime().longValue());
+        }
+        assertEquals(null, newLifecycleState.getFailedStep());
+        assertEquals(null, newLifecycleState.getStepInfo());
+    }
+
+    public static void assertClusterStateOnNextStep(ClusterState oldClusterState, Index index, StepKey currentStep, StepKey nextStep,
+            ClusterState newClusterState, long now) {
+        assertNotSame(oldClusterState, newClusterState);
+        MetaData newMetadata = newClusterState.metaData();
+        assertNotSame(oldClusterState.metaData(), newMetadata);
+        IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
+        assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(index));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
+        assertNotSame(oldLifecycleState, newLifecycleState);
+        assertEquals(nextStep.getPhase(), newLifecycleState.getPhase());
+        assertEquals(nextStep.getAction(), newLifecycleState.getAction());
+        assertEquals(nextStep.getName(), newLifecycleState.getStep());
+        if (currentStep.getPhase().equals(nextStep.getPhase())) {
+            assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
+        } else {
+            assertEquals(now, newLifecycleState.getPhaseTime().longValue());
+        }
+        if (currentStep.getAction().equals(nextStep.getAction())) {
+            assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
+        } else {
+            assertEquals(now, newLifecycleState.getActionTime().longValue());
+        }
+        assertEquals(now, newLifecycleState.getStepTime().longValue());
+        assertEquals(null, newLifecycleState.getFailedStep());
+        assertEquals(null, newLifecycleState.getStepInfo());
+    }
+
+    private void assertClusterStateOnErrorStep(ClusterState oldClusterState, Index index, StepKey currentStep,
+                                               ClusterState newClusterState, long now, String expectedCauseValue) throws IOException {
+        assertNotSame(oldClusterState, newClusterState);
+        MetaData newMetadata = newClusterState.metaData();
+        assertNotSame(oldClusterState.metaData(), newMetadata);
+        IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
+        assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(index));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
+        assertNotSame(oldLifecycleState, newLifecycleState);
+        assertEquals(currentStep.getPhase(), newLifecycleState.getPhase());
+        assertEquals(currentStep.getAction(), newLifecycleState.getAction());
+        assertEquals(ErrorStep.NAME, newLifecycleState.getStep());
+        assertEquals(currentStep.getName(), newLifecycleState.getFailedStep());
+        assertEquals(expectedCauseValue, newLifecycleState.getStepInfo());
+        assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
+        assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
+        assertEquals(now, newLifecycleState.getStepTime().longValue());
+    }
+
+    private void assertClusterStateStepInfo(ClusterState oldClusterState, Index index, StepKey currentStep, ClusterState newClusterState,
+            ToXContentObject stepInfo) throws IOException {
+        XContentBuilder stepInfoXContentBuilder = JsonXContent.contentBuilder();
+        stepInfo.toXContent(stepInfoXContentBuilder, ToXContent.EMPTY_PARAMS);
+        String expectedstepInfoValue = BytesReference.bytes(stepInfoXContentBuilder).utf8ToString();
+        assertNotSame(oldClusterState, newClusterState);
+        MetaData newMetadata = newClusterState.metaData();
+        assertNotSame(oldClusterState.metaData(), newMetadata);
+        IndexMetaData newIndexMetadata = newMetadata.getIndexSafe(index);
+        assertNotSame(oldClusterState.metaData().index(index), newIndexMetadata);
+        LifecycleExecutionState newLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(newClusterState.metaData().index(index));
+        LifecycleExecutionState oldLifecycleState = LifecycleExecutionState
+            .fromIndexMetadata(oldClusterState.metaData().index(index));
+        assertNotSame(oldLifecycleState, newLifecycleState);
+        assertEquals(currentStep.getPhase(), newLifecycleState.getPhase());
+        assertEquals(currentStep.getAction(), newLifecycleState.getAction());
+        assertEquals(currentStep.getName(), newLifecycleState.getStep());
+        assertEquals(expectedstepInfoValue, newLifecycleState.getStepInfo());
+        assertEquals(oldLifecycleState.getPhaseTime(), newLifecycleState.getPhaseTime());
+        assertEquals(oldLifecycleState.getActionTime(), newLifecycleState.getActionTime());
+        assertEquals(newLifecycleState.getStepTime(), newLifecycleState.getStepTime());
+    }
+
+    private static class MockAsyncActionStep extends AsyncActionStep {
+
+        private Exception exception;
+        private boolean willComplete;
+        private boolean indexSurvives = true;
+        private long executeCount = 0;
+        private CountDownLatch latch;
+
+        MockAsyncActionStep(StepKey key, StepKey nextStepKey) {
+            super(key, nextStepKey, null);
+        }
+
+        void setException(Exception exception) {
+            this.exception = exception;
+        }
+
+        @Override
+        public boolean indexSurvives() {
+            return indexSurvives;
+        }
+
+        void setWillComplete(boolean willComplete) {
+            this.willComplete = willComplete;
+        }
+
+        long getExecuteCount() {
+            return executeCount;
+        }
+
+        public void setLatch(CountDownLatch latch) {
+            this.latch = latch;
+        }
+
+        @Override
+        public void performAction(IndexMetaData indexMetaData, ClusterState currentState, Listener listener) {
+            executeCount++;
+            if (latch != null) {
+                latch.countDown();
+            }
+            if (exception == null) {
+                listener.onResponse(willComplete);
+            } else {
+                listener.onFailure(exception);
+            }
+        }
+
+    }
+
+    private static class MockAsyncWaitStep extends AsyncWaitStep {
+
+        private Exception exception;
+        private boolean willComplete;
+        private long executeCount = 0;
+        private ToXContentObject expectedInfo = null;
+        private CountDownLatch latch;
+
+        MockAsyncWaitStep(StepKey key, StepKey nextStepKey) {
+            super(key, nextStepKey, null);
+        }
+
+        void setException(Exception exception) {
+            this.exception = exception;
+        }
+
+        long getExecuteCount() {
+            return executeCount;
+        }
+
+        public void setLatch(CountDownLatch latch) {
+            this.latch = latch;
+        }
+
+        @Override
+        public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
+            executeCount++;
+            if (latch != null) {
+                latch.countDown();
+            }
+            if (exception == null) {
+                listener.onResponse(willComplete, expectedInfo);
+            } else {
+                listener.onFailure(exception);
+            }
+        }
+
+    }
+
+    static class MockClusterStateActionStep extends ClusterStateActionStep {
+
+        private RuntimeException exception;
+        private long executeCount = 0;
+        private CountDownLatch latch;
+
+        MockClusterStateActionStep(StepKey key, StepKey nextStepKey) {
+            super(key, nextStepKey);
+        }
+
+        public void setException(RuntimeException exception) {
+            this.exception = exception;
+        }
+
+        public void setLatch(CountDownLatch latch) {
+            this.latch = latch;
+        }
+
+        public long getExecuteCount() {
+            return executeCount;
+        }
+
+        @Override
+        public ClusterState performAction(Index index, ClusterState clusterState) {
+            executeCount++;
+            if (latch != null) {
+                latch.countDown();
+            }
+            if (exception != null) {
+                throw exception;
+            }
+            return clusterState;
+        }
+    }
+
+    static class MockClusterStateWaitStep extends ClusterStateWaitStep {
+
+        private RuntimeException exception;
+        private boolean willComplete;
+        private long executeCount = 0;
+        private ToXContentObject expectedInfo = null;
+
+        MockClusterStateWaitStep(StepKey key, StepKey nextStepKey) {
+            super(key, nextStepKey);
+        }
+
+        public void setException(RuntimeException exception) {
+            this.exception = exception;
+        }
+
+        public void setWillComplete(boolean willComplete) {
+            this.willComplete = willComplete;
+        }
+
+        void expectedInfo(ToXContentObject expectedInfo) {
+            this.expectedInfo = expectedInfo;
+        }
+
+        public long getExecuteCount() {
+            return executeCount;
+        }
+
+        @Override
+        public Result isConditionMet(Index index, ClusterState clusterState) {
+            executeCount++;
+            if (exception != null) {
+                throw exception;
+            }
+            return new Result(willComplete, expectedInfo);
+        }
+
+    }
+
+    private static class SetStepInfoUpdateTaskMatcher extends ArgumentMatcher<SetStepInfoUpdateTask> {
+
+        private Index index;
+        private String policy;
+        private StepKey currentStepKey;
+        private ToXContentObject stepInfo;
+
+        SetStepInfoUpdateTaskMatcher(Index index, String policy, StepKey currentStepKey, ToXContentObject stepInfo) {
+            this.index = index;
+            this.policy = policy;
+            this.currentStepKey = currentStepKey;
+            this.stepInfo = stepInfo;
+        }
+
+        @Override
+        public boolean matches(Object argument) {
+            if (argument == null || argument instanceof SetStepInfoUpdateTask == false) {
+                return false;
+            }
+            SetStepInfoUpdateTask task = (SetStepInfoUpdateTask) argument;
+            return Objects.equals(index, task.getIndex()) &&
+                    Objects.equals(policy, task.getPolicy())&&
+                    Objects.equals(currentStepKey, task.getCurrentStepKey()) &&
+                    Objects.equals(xContentToString(stepInfo), xContentToString(task.getStepInfo()));
+        }
+
+        private String xContentToString(ToXContentObject xContent) {
+            try {
+                XContentBuilder builder = JsonXContent.contentBuilder();
+                stepInfo.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                return BytesReference.bytes(builder).utf8ToString();
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+    }
+
+    private static class ExecuteStepsUpdateTaskMatcher extends ArgumentMatcher<ExecuteStepsUpdateTask> {
+
+        private Index index;
+        private String policy;
+        private Step startStep;
+
+        ExecuteStepsUpdateTaskMatcher(Index index, String policy, Step startStep) {
+            this.index = index;
+            this.policy = policy;
+            this.startStep = startStep;
+        }
+
+        @Override
+        public boolean matches(Object argument) {
+            if (argument == null || argument instanceof ExecuteStepsUpdateTask == false) {
+                return false;
+            }
+            ExecuteStepsUpdateTask task = (ExecuteStepsUpdateTask) argument;
+            return Objects.equals(index, task.getIndex()) &&
+                    Objects.equals(policy, task.getPolicy()) &&
+                    Objects.equals(startStep, task.getStartStep());
+        }
+
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java
new file mode 100644
index 0000000000000..13fe9c1c69002
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.MockAction;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Collections;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutorService;
+
+import static org.elasticsearch.node.Node.NODE_MASTER_SETTING;
+import static org.elasticsearch.xpack.core.indexlifecycle.AbstractStepTestCase.randomStepKey;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndexLifecycleServiceTests extends ESTestCase {
+
+    private ClusterService clusterService;
+    private IndexLifecycleService indexLifecycleService;
+    private String nodeId;
+    private DiscoveryNode masterNode;
+    private IndicesAdminClient indicesClient;
+    private long now;
+
+    @Before
+    public void prepareServices() {
+        nodeId = randomAlphaOfLength(10);
+        ExecutorService executorService = mock(ExecutorService.class);
+        clusterService = mock(ClusterService.class);
+        masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        now = randomNonNegativeLong();
+        Clock clock = Clock.fixed(Instant.ofEpochMilli(now), ZoneId.of(randomFrom(ZoneId.getAvailableZoneIds())));
+
+        doAnswer(invocationOnMock -> null).when(clusterService).addListener(any());
+        doAnswer(invocationOnMock -> {
+            Runnable runnable = (Runnable) invocationOnMock.getArguments()[0];
+            runnable.run();
+            return null;
+        }).when(executorService).execute(any());
+        Settings settings = Settings.builder().put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s").build();
+        when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings,
+            Collections.singleton(LifecycleSettings.LIFECYCLE_POLL_INTERVAL_SETTING)));
+
+        Client client = mock(Client.class);
+        AdminClient adminClient = mock(AdminClient.class);
+        indicesClient = mock(IndicesAdminClient.class);
+        when(client.admin()).thenReturn(adminClient);
+        when(adminClient.indices()).thenReturn(indicesClient);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+
+        indexLifecycleService = new IndexLifecycleService(Settings.EMPTY, client, clusterService, clock, () -> now, null);
+        Mockito.verify(clusterService).addListener(indexLifecycleService);
+        Mockito.verify(clusterService).addStateApplier(indexLifecycleService);
+    }
+
+    @After
+    public void cleanup() {
+        indexLifecycleService.close();
+    }
+
+
+    public void testStoppedModeSkip() {
+        String policyName = randomAlphaOfLengthBetween(1, 20);
+        IndexLifecycleRunnerTests.MockClusterStateActionStep mockStep =
+            new IndexLifecycleRunnerTests.MockClusterStateActionStep(randomStepKey(), randomStepKey());
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = newTestLifecyclePolicy(policyName, Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+            .fPut(index.getName(), indexMetadata);
+        MetaData metaData = MetaData.builder()
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.STOPPED))
+            .indices(indices.build())
+            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
+        indexLifecycleService.applyClusterState(event);
+        indexLifecycleService.triggerPolicies(currentState, randomBoolean());
+        assertThat(mockStep.getExecuteCount(), equalTo(0L));
+    }
+
+    public void testRequestedStopOnShrink() {
+        Step.StepKey mockShrinkStep = new Step.StepKey(randomAlphaOfLength(4), ShrinkAction.NAME, randomAlphaOfLength(5));
+        String policyName = randomAlphaOfLengthBetween(1, 20);
+        IndexLifecycleRunnerTests.MockClusterStateActionStep mockStep =
+            new IndexLifecycleRunnerTests.MockClusterStateActionStep(mockShrinkStep, randomStepKey());
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = newTestLifecyclePolicy(policyName, Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(mockShrinkStep.getPhase());
+        lifecycleState.setAction(mockShrinkStep.getAction());
+        lifecycleState.setStep(mockShrinkStep.getName());
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+            .fPut(index.getName(), indexMetadata);
+        MetaData metaData = MetaData.builder()
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.STOPPING))
+            .indices(indices.build())
+            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
+        SetOnce<Boolean> executedShrink = new SetOnce<>();
+        doAnswer(invocationOnMock -> {
+            executedShrink.set(true);
+            return null;
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(ExecuteStepsUpdateTask.class));
+        indexLifecycleService.applyClusterState(event);
+        indexLifecycleService.triggerPolicies(currentState, true);
+        assertTrue(executedShrink.get());
+    }
+
+    public void testRequestedStopOnSafeAction() {
+        String policyName = randomAlphaOfLengthBetween(1, 20);
+        Step.StepKey currentStepKey = randomStepKey();
+        IndexLifecycleRunnerTests.MockClusterStateActionStep mockStep =
+            new IndexLifecycleRunnerTests.MockClusterStateActionStep(currentStepKey, randomStepKey());
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = newTestLifecyclePolicy(policyName, Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(currentStepKey.getPhase());
+        lifecycleState.setAction(currentStepKey.getAction());
+        lifecycleState.setStep(currentStepKey.getName());
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+            .fPut(index.getName(), indexMetadata);
+        MetaData metaData = MetaData.builder()
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.STOPPING))
+            .indices(indices.build())
+            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
+
+        SetOnce<Boolean> ranPolicy = new SetOnce<>();
+        SetOnce<Boolean> moveToMaintenance = new SetOnce<>();
+        doAnswer(invocationOnMock -> {
+            ranPolicy.set(true);
+            throw new AssertionError("invalid invocation");
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(ExecuteStepsUpdateTask.class));
+
+        doAnswer(invocationOnMock -> {
+            OperationModeUpdateTask task = (OperationModeUpdateTask) invocationOnMock.getArguments()[1];
+            assertThat(task.getOperationMode(), equalTo(OperationMode.STOPPED));
+            moveToMaintenance.set(true);
+            return null;
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(OperationModeUpdateTask.class));
+
+        indexLifecycleService.applyClusterState(event);
+        indexLifecycleService.triggerPolicies(currentState, randomBoolean());
+        assertNull(ranPolicy.get());
+        assertTrue(moveToMaintenance.get());
+    }
+
+    public void testTriggeredDifferentJob() {
+        Mockito.reset(clusterService);
+        SchedulerEngine.Event schedulerEvent = new SchedulerEngine.Event("foo", randomLong(), randomLong());
+        indexLifecycleService.triggered(schedulerEvent);
+        Mockito.verifyZeroInteractions(indicesClient, clusterService);
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicyClientTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicyClientTests.java
new file mode 100644
index 0000000000000..4fec7ba80db8e
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicyClientTests.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LifecyclePolicyClientTests extends ESTestCase {
+
+    public void testExecuteWithHeadersAsyncNoHeaders() throws InterruptedException {
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+
+        final CountDownLatch latch = new CountDownLatch(2);
+        final ActionListener<SearchResponse> listener = ActionListener.wrap(v -> {
+            assertTrue(threadContext.getHeaders().isEmpty());
+            latch.countDown();
+        }, e -> fail(e.getMessage()));
+
+        doAnswer(invocationOnMock -> {
+            assertTrue(threadContext.getHeaders().isEmpty());
+            latch.countDown();
+            ((ActionListener<?>) invocationOnMock.getArguments()[2]).onResponse(null);
+            return null;
+        }).when(client).execute(anyObject(), anyObject(), anyObject());
+
+        SearchRequest request = new SearchRequest("foo");
+
+        try (LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client, ClientHelper.INDEX_LIFECYCLE_ORIGIN,
+                Collections.emptyMap())) {
+            policyClient.execute(SearchAction.INSTANCE, request, listener);
+        }
+
+        latch.await();
+    }
+
+    public void testExecuteWithHeadersAsyncWrongHeaders() throws InterruptedException {
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+
+        final CountDownLatch latch = new CountDownLatch(2);
+        final ActionListener<SearchResponse> listener = ActionListener.wrap(v -> {
+            assertTrue(threadContext.getHeaders().isEmpty());
+            latch.countDown();
+        }, e -> fail(e.getMessage()));
+
+        doAnswer(invocationOnMock -> {
+            assertTrue(threadContext.getHeaders().isEmpty());
+            latch.countDown();
+            ((ActionListener<?>) invocationOnMock.getArguments()[2]).onResponse(null);
+            return null;
+        }).when(client).execute(anyObject(), anyObject(), anyObject());
+
+        SearchRequest request = new SearchRequest("foo");
+        Map<String, String> headers = new HashMap<>(1);
+        headers.put("foo", "foo");
+        headers.put("bar", "bar");
+
+        try (LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client, ClientHelper.INDEX_LIFECYCLE_ORIGIN,
+                headers)) {
+            policyClient.execute(SearchAction.INSTANCE, request, listener);
+        }
+
+        latch.await();
+    }
+
+    public void testExecuteWithHeadersAsyncWithHeaders() throws Exception {
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        when(client.threadPool()).thenReturn(threadPool);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+
+        final CountDownLatch latch = new CountDownLatch(2);
+        final ActionListener<SearchResponse> listener = ActionListener.wrap(v -> {
+            assertTrue(threadContext.getHeaders().isEmpty());
+            latch.countDown();
+        }, e -> fail(e.getMessage()));
+
+        doAnswer(invocationOnMock -> {
+            assertThat(threadContext.getHeaders().size(), equalTo(2));
+            assertThat(threadContext.getHeaders().get("es-security-runas-user"), equalTo("foo"));
+            assertThat(threadContext.getHeaders().get("_xpack_security_authentication"), equalTo("bar"));
+            latch.countDown();
+            ((ActionListener<?>) invocationOnMock.getArguments()[2]).onResponse(null);
+            return null;
+        }).when(client).execute(anyObject(), anyObject(), anyObject());
+
+        SearchRequest request = new SearchRequest("foo");
+        Map<String, String> headers = new HashMap<>(1);
+        headers.put("es-security-runas-user", "foo");
+        headers.put("_xpack_security_authentication", "bar");
+
+        try (LifecyclePolicySecurityClient policyClient = new LifecyclePolicySecurityClient(client, ClientHelper.INDEX_LIFECYCLE_ORIGIN,
+                headers)) {
+            policyClient.execute(SearchAction.INSTANCE, request, listener);
+        }
+
+        latch.await();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LockableLifecycleType.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LockableLifecycleType.java
new file mode 100644
index 0000000000000..3e09133c435a8
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/LockableLifecycleType.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This {@link LifecycleType} is used for encapsulating test policies
+ * used in integration tests where the underlying {@link LifecycleAction}s are
+ * able to communicate with the test
+ */
+public class LockableLifecycleType implements LifecycleType {
+    public static final String TYPE = "lockable";
+    public static final LockableLifecycleType INSTANCE = new LockableLifecycleType();
+
+    @Override
+    public List<Phase> getOrderedPhases(Map<String, Phase> phases) {
+        return new ArrayList<>(phases.values());
+    }
+
+    @Override
+    public String getNextPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        return null;
+    }
+
+    @Override
+    public String getPreviousPhaseName(String currentPhaseName, Map<String, Phase> phases) {
+        return null;
+    }
+
+    @Override
+    public List<LifecycleAction> getOrderedActions(Phase phase) {
+        return new ArrayList<>(phase.getActions().values());
+    }
+
+    @Override
+    public String getNextActionName(String currentActionName, Phase phase) {
+        return null;
+    }
+
+    @Override
+    public void validate(Collection<Phase> phases) {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) {
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java
new file mode 100644
index 0000000000000..dc3a6602f39ba
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToErrorStepUpdateTaskTests.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class MoveToErrorStepUpdateTaskTests extends ESTestCase {
+
+    String policy;
+    ClusterState clusterState;
+    Index index;
+
+    @Before
+    public void setupClusterState() {
+        policy = randomAlphaOfLength(10);
+        LifecyclePolicy lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy);
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        index = indexMetadata.getIndex();
+        IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(
+            Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong())),
+            OperationMode.RUNNING);
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta)
+            .build();
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+    }
+
+    public void testExecuteSuccessfullyMoved() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        long now = randomNonNegativeLong();
+        Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
+
+        setStateToKey(currentStepKey);
+
+        MoveToErrorStepUpdateTask task = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(actualKey, equalTo(new StepKey(currentStepKey.getPhase(), currentStepKey.getAction(), ErrorStep.NAME)));
+        assertThat(lifecycleState.getFailedStep(), equalTo(currentStepKey.getName()));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
+
+        XContentBuilder causeXContentBuilder = JsonXContent.contentBuilder();
+        causeXContentBuilder.startObject();
+        ElasticsearchException.generateThrowableXContent(causeXContentBuilder, ToXContent.EMPTY_PARAMS, cause);
+        causeXContentBuilder.endObject();
+        String expectedCauseValue = BytesReference.bytes(causeXContentBuilder).utf8ToString();
+        assertThat(lifecycleState.getStepInfo(), equalTo(expectedCauseValue));
+    }
+
+    public void testExecuteNoopDifferentStep() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        StepKey notCurrentStepKey = new StepKey("not-current", "not-current", "not-current");
+        long now = randomNonNegativeLong();
+        Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
+        setStateToKey(notCurrentStepKey);
+        MoveToErrorStepUpdateTask task = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        assertThat(newState, sameInstance(clusterState));
+    }
+
+    public void testExecuteNoopDifferentPolicy() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        long now = randomNonNegativeLong();
+        Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
+        setStateToKey(currentStepKey);
+        setStatePolicy("not-" + policy);
+        MoveToErrorStepUpdateTask task = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause, () -> now);
+        ClusterState newState = task.execute(clusterState);
+        assertThat(newState, sameInstance(clusterState));
+    }
+
+    public void testOnFailure() {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        long now = randomNonNegativeLong();
+        Exception cause = new ElasticsearchException("THIS IS AN EXPECTED CAUSE");
+
+        setStateToKey(currentStepKey);
+
+        MoveToErrorStepUpdateTask task = new MoveToErrorStepUpdateTask(index, policy, currentStepKey, cause, () -> now);
+        Exception expectedException = new RuntimeException();
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> task.onFailure(randomAlphaOfLength(10), expectedException));
+        assertEquals("policy [" + policy + "] for index [" + index.getName() + "] failed trying to move from step [" + currentStepKey
+                + "] to the ERROR step.", exception.getMessage());
+        assertSame(expectedException, exception.getCause());
+    }
+
+    private void setStatePolicy(String policy) {
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.metaData())
+                .updateSettings(Settings.builder()
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), index.getName())).build();
+
+    }
+    private void setStateToKey(StepKey stepKey) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
+
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.getMetaData())
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java
new file mode 100644
index 0000000000000..f166bba25c986
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/MoveToNextStepUpdateTaskTests.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.equalTo;
+
+public class MoveToNextStepUpdateTaskTests extends ESTestCase {
+
+    String policy;
+    ClusterState clusterState;
+    Index index;
+    LifecyclePolicy lifecyclePolicy;
+
+    @Before
+    public void setupClusterState() {
+        policy = randomAlphaOfLength(10);
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        index = indexMetadata.getIndex();
+        lifecyclePolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policy);
+        IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(
+            Collections.singletonMap(policy, new LifecyclePolicyMetadata(lifecyclePolicy, Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong())),
+            OperationMode.RUNNING);
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta)
+            .build();
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+    }
+
+    public void testExecuteSuccessfullyMoved() {
+        long now = randomNonNegativeLong();
+        List<Step> steps = lifecyclePolicy.toSteps(null);
+        StepKey currentStepKey = steps.get(0).getKey();
+        StepKey nextStepKey = steps.get(0).getNextStepKey();
+
+        setStateToKey(currentStepKey, now);
+
+        AtomicBoolean changed = new AtomicBoolean(false);
+        MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, nextStepKey,
+            () -> now, state -> changed.set(true));
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(actualKey, equalTo(nextStepKey));
+        assertThat(lifecycleState.getPhaseTime(), equalTo(now));
+        assertThat(lifecycleState.getActionTime(), equalTo(now));
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
+        task.clusterStateProcessed("source", clusterState, newState);
+        assertTrue(changed.get());
+    }
+
+    public void testExecuteDifferentCurrentStep() {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        StepKey notCurrentStepKey = new StepKey("not-current", "not-current", "not-current");
+        long now = randomNonNegativeLong();
+        setStateToKey(notCurrentStepKey, now);
+        MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, null);
+        ClusterState newState = task.execute(clusterState);
+        assertSame(newState, clusterState);
+    }
+
+    public void testExecuteDifferentPolicy() {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        long now = randomNonNegativeLong();
+        setStateToKey(currentStepKey, now);
+        setStatePolicy("not-" + policy);
+        MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, null, () -> now, null);
+        ClusterState newState = task.execute(clusterState);
+        assertSame(newState, clusterState);
+    }
+
+    public void testExecuteSuccessfulMoveWithInvalidNextStep() {
+        long now = randomNonNegativeLong();
+        List<Step> steps = lifecyclePolicy.toSteps(null);
+        StepKey currentStepKey = steps.get(0).getKey();
+        StepKey invalidNextStep = new StepKey("next-invalid", "next-invalid", "next-invalid");
+
+        setStateToKey(currentStepKey, now);
+
+        SetOnce<Boolean> changed = new SetOnce<>();
+        MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey,
+            invalidNextStep, () -> now, s -> changed.set(true));
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(actualKey, equalTo(invalidNextStep));
+        assertThat(lifecycleState.getPhaseTime(), equalTo(now));
+        assertThat(lifecycleState.getActionTime(), equalTo(now));
+        assertThat(lifecycleState.getStepTime(), equalTo(now));
+        task.clusterStateProcessed("source", clusterState, newState);
+        assertTrue(changed.get());
+    }
+
+    public void testOnFailure() {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        StepKey nextStepKey = new StepKey("next-phase", "next-action", "next-name");
+        long now = randomNonNegativeLong();
+
+        setStateToKey(currentStepKey, now);
+
+        MoveToNextStepUpdateTask task = new MoveToNextStepUpdateTask(index, policy, currentStepKey, nextStepKey, () -> now, state -> {});
+        Exception expectedException = new RuntimeException();
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> task.onFailure(randomAlphaOfLength(10), expectedException));
+        assertEquals("policy [" + policy + "] for index [" + index.getName() + "] failed trying to move from step [" + currentStepKey
+                + "] to step [" + nextStepKey + "].", exception.getMessage());
+        assertSame(expectedException, exception.getCause());
+    }
+
+    private void setStatePolicy(String policy) {
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.metaData())
+                .updateSettings(Settings.builder()
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), index.getName())).build();
+
+    }
+    private void setStateToKey(StepKey stepKey, long now) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setPhaseTime(now);
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setActionTime(now);
+        lifecycleState.setStep(stepKey.getName());
+        lifecycleState.setStepTime(now);
+        lifecycleState.setPhaseDefinition("{\"actions\":{\"TEST_ACTION\":{}}}");
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.getMetaData())
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTaskTests.java
new file mode 100644
index 0000000000000..dccd12e15f114
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/OperationModeUpdateTaskTests.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class OperationModeUpdateTaskTests extends ESTestCase {
+
+    public void testExecute() {
+        assertMove(OperationMode.RUNNING, OperationMode.STOPPING);
+        assertMove(OperationMode.STOPPING, randomFrom(OperationMode.RUNNING, OperationMode.STOPPED));
+        assertMove(OperationMode.STOPPED, OperationMode.RUNNING);
+
+        OperationMode mode = randomFrom(OperationMode.values());
+        assertNoMove(mode, mode);
+        assertNoMove(OperationMode.STOPPED, OperationMode.STOPPING);
+        assertNoMove(OperationMode.RUNNING, OperationMode.STOPPED);
+    }
+
+    public void testExecuteWithEmptyMetadata() {
+        OperationMode requestedMode = OperationMode.STOPPING;
+        OperationMode newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(),
+            requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+
+        requestedMode = randomFrom(OperationMode.RUNNING, OperationMode.STOPPED);
+        newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(),
+            requestedMode, false);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+    }
+
+    private void assertMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+    }
+
+    private void assertNoMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, true);
+        assertThat(newMode, equalTo(currentMode));
+    }
+
+    private OperationMode executeUpdate(boolean metadataInstalled, OperationMode currentMode, OperationMode requestMode,
+                                        boolean assertSameClusterState) {
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), currentMode);
+        ImmutableOpenMap.Builder<String, MetaData.Custom> customsMapBuilder = ImmutableOpenMap.builder();
+        MetaData.Builder metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build());
+        if (metadataInstalled) {
+            metaData.customs(customsMapBuilder.fPut(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata).build());
+        }
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        OperationModeUpdateTask task = new OperationModeUpdateTask(requestMode);
+        ClusterState newState = task.execute(state);
+        if (assertSameClusterState) {
+            assertSame(state, newState);
+        } else {
+            assertThat(state, not(equalTo(newState)));
+        }
+        IndexLifecycleMetadata newMetaData = newState.metaData().custom(IndexLifecycleMetadata.TYPE);
+        assertThat(newMetaData.getPolicyMetadatas(), equalTo(indexLifecycleMetadata.getPolicyMetadatas()));
+        return newMetaData.getOperationMode();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PhaseStatsTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PhaseStatsTests.java
new file mode 100644
index 0000000000000..fe7fd1fca05d3
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PhaseStatsTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class PhaseStatsTests extends AbstractWireSerializingTestCase<PhaseStats> {
+
+    @Override
+    protected PhaseStats createTestInstance() {
+        return randomPhaseStats();
+    }
+
+    static PhaseStats randomPhaseStats() {
+        TimeValue minimumAge = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after");
+        String[] actionNames = generateRandomStringArray(10, 20, false);
+        return new PhaseStats(minimumAge, actionNames);
+    }
+
+    @Override
+    protected PhaseStats mutateInstance(PhaseStats instance) throws IOException {
+        TimeValue minimumAge = instance.getAfter();
+        String[] actionNames = instance.getActionNames();
+        switch (between(0, 1)) {
+        case 0:
+            minimumAge = randomValueOtherThan(minimumAge,
+                    () -> TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"));
+            break;
+        case 1:
+            actionNames = Arrays.copyOf(actionNames, actionNames.length + 1);
+            actionNames[actionNames.length - 1] = randomAlphaOfLengthBetween(10, 20);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new PhaseStats(minimumAge, actionNames);
+    }
+
+    @Override
+    protected Reader<PhaseStats> instanceReader() {
+        return PhaseStats::new;
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStatsTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStatsTests.java
new file mode 100644
index 0000000000000..5ced745c2fb3f
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStatsTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class PolicyStatsTests extends AbstractWireSerializingTestCase<PolicyStats> {
+
+    @Override
+    protected PolicyStats createTestInstance() {
+        return randomPolicyStats();
+    }
+
+    static PolicyStats randomPolicyStats() {
+        Map<String, PhaseStats> phaseStats = new HashMap<>();
+        int size = randomIntBetween(0, 10);
+        for (int i = 0; i < size; i++) {
+            phaseStats.put(randomAlphaOfLength(10), PhaseStatsTests.randomPhaseStats());
+        }
+        int numberIndicesManaged = randomIntBetween(0, 1000);
+        return new PolicyStats(phaseStats, numberIndicesManaged);
+    }
+
+    @Override
+    protected PolicyStats mutateInstance(PolicyStats instance) throws IOException {
+        Map<String, PhaseStats> phaseStats = instance.getPhaseStats();
+        int numberIndicesManaged = instance.getIndicesManaged();
+        switch (between(0, 1)) {
+        case 0:
+            phaseStats = new HashMap<>(phaseStats);
+            phaseStats.put(randomAlphaOfLength(11), PhaseStatsTests.randomPhaseStats());
+            break;
+        case 1:
+            numberIndicesManaged += randomIntBetween(1, 10);
+            break;
+        default:
+            throw new AssertionError("Illegal randomisation branch");
+        }
+        return new PolicyStats(phaseStats, numberIndicesManaged);
+    }
+
+    @Override
+    protected Reader<PolicyStats> instanceReader() {
+        return PolicyStats::new;
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java
new file mode 100644
index 0000000000000..611522a59b0d4
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/PolicyStepsRegistryTests.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.ErrorStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.InitializePolicyContextStep;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyTests;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.MockStep;
+import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
+import org.elasticsearch.xpack.core.indexlifecycle.Phase;
+import org.elasticsearch.xpack.core.indexlifecycle.PhaseExecutionInfo;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkStep;
+import org.elasticsearch.xpack.core.indexlifecycle.Step;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+
+public class PolicyStepsRegistryTests extends ESTestCase {
+    private static final Step.StepKey MOCK_STEP_KEY = new Step.StepKey("mock", "mock", "mock");
+    private static final NamedXContentRegistry REGISTRY = new NamedXContentRegistry(new IndexLifecycle(Settings.EMPTY).getNamedXContent());
+
+    private IndexMetaData emptyMetaData(Index index) {
+        return IndexMetaData.builder(index.getName()).settings(settings(Version.CURRENT))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+    }
+
+    public void testGetFirstStep() {
+        String policyName = randomAlphaOfLengthBetween(2, 10);
+        Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null);
+        Map<String, Step> firstStepMap = Collections.singletonMap(policyName, expectedFirstStep);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, NamedXContentRegistry.EMPTY, null);
+        Step actualFirstStep = registry.getFirstStep(policyName);
+        assertThat(actualFirstStep, sameInstance(expectedFirstStep));
+    }
+
+    public void testGetFirstStepUnknownPolicy() {
+        String policyName = randomAlphaOfLengthBetween(2, 10);
+        Step expectedFirstStep = new MockStep(MOCK_STEP_KEY, null);
+        Map<String, Step> firstStepMap = Collections.singletonMap(policyName, expectedFirstStep);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, firstStepMap, null, NamedXContentRegistry.EMPTY, null);
+        Step actualFirstStep = registry.getFirstStep(policyName + "unknown");
+        assertNull(actualFirstStep);
+    }
+
+    public void testGetStep() {
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        LifecyclePolicy policy = LifecyclePolicyTests.randomTimeseriesLifecyclePolicyWithAllPhases("policy");
+        LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), 1, randomNonNegativeLong());
+        String phaseName = randomFrom(policy.getPhases().keySet());
+        Phase phase = policy.getPhases().get(phaseName);
+        PhaseExecutionInfo pei = new PhaseExecutionInfo(policy.getName(), phase, 1, randomNonNegativeLong());
+        String phaseJson = Strings.toString(pei);
+        LifecycleAction action = randomFrom(phase.getActions().values());
+        Step step = randomFrom(action.toSteps(client, phaseName, MOCK_STEP_KEY));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0)
+                .put("index.version.created", Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, "policy")
+                .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
+        SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
+        metas.put("policy", policyMetadata);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(metas, null, null, REGISTRY, client);
+        Step actualStep = registry.getStep(indexMetaData, step.getKey());
+        assertThat(actualStep.getKey(), equalTo(step.getKey()));
+    }
+
+    public void testGetStepErrorStep() {
+        Step.StepKey errorStepKey = new Step.StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10), ErrorStep.NAME);
+        Step expectedStep = new ErrorStep(errorStepKey);
+        Index index = new Index("test", "uuid");
+        Map<Index, List<Step>> indexSteps = Collections.singletonMap(index, Collections.singletonList(expectedStep));
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, NamedXContentRegistry.EMPTY, null);
+        Step actualStep = registry.getStep(emptyMetaData(index), errorStepKey);
+        assertThat(actualStep, equalTo(expectedStep));
+    }
+
+    public void testGetStepUnknownPolicy() {
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(null, null, null, NamedXContentRegistry.EMPTY, null);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> registry.getStep(emptyMetaData(new Index("test", "uuid")), MOCK_STEP_KEY));
+        assertThat(e.getMessage(),
+            containsString("failed to retrieve step {\"phase\":\"mock\",\"action\":\"mock\",\"name\":\"mock\"}" +
+                " as index [test] has no policy"));
+    }
+
+    public void testGetStepForIndexWithNoPhaseGetsInitializationStep() {
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        LifecyclePolicy policy = LifecyclePolicyTests.randomTimeseriesLifecyclePolicy("policy");
+        LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), 1, randomNonNegativeLong());
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0)
+                .put("index.version.created", Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, "policy")
+                .build())
+            .build();
+        SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
+        metas.put("policy", policyMetadata);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(metas, null, null, REGISTRY, client);
+        Step step = registry.getStep(indexMetaData, InitializePolicyContextStep.KEY);
+        assertNotNull(step);
+    }
+
+    public void testGetStepUnknownStepKey() {
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        LifecyclePolicy policy = LifecyclePolicyTests.randomTimeseriesLifecyclePolicyWithAllPhases("policy");
+        LifecyclePolicyMetadata policyMetadata = new LifecyclePolicyMetadata(policy, Collections.emptyMap(), 1, randomNonNegativeLong());
+        String phaseName = randomFrom(policy.getPhases().keySet());
+        Phase phase = policy.getPhases().get(phaseName);
+        PhaseExecutionInfo pei = new PhaseExecutionInfo(policy.getName(), phase, 1, randomNonNegativeLong());
+        String phaseJson = Strings.toString(pei);
+        LifecycleAction action = randomFrom(phase.getActions().values());
+        Step step = randomFrom(action.toSteps(client, phaseName, MOCK_STEP_KEY));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhaseDefinition(phaseJson);
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(Settings.builder()
+                .put("index.number_of_shards", 1)
+                .put("index.number_of_replicas", 0)
+                .put("index.version.created", Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, "policy")
+                .build())
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .build();
+        SortedMap<String, LifecyclePolicyMetadata> metas = new TreeMap<>();
+        metas.put("policy", policyMetadata);
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(metas, null, null, REGISTRY, client);
+        Step actualStep = registry.getStep(indexMetaData,
+            new Step.StepKey(step.getKey().getPhase(), step.getKey().getAction(), step.getKey().getName() + "-bad"));
+        assertNull(actualStep);
+    }
+
+    public void testUpdateFromNothingToSomethingToNothing() throws Exception {
+        Index index = new Index("test", "uuid");
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        String policyName = randomAlphaOfLength(5);
+        LifecyclePolicy newPolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policyName);
+        logger.info("--> policy: {}", newPolicy);
+        List<Step> policySteps = newPolicy.toSteps(client);
+        Map<String, String> headers = new HashMap<>();
+        if (randomBoolean()) {
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        }
+        Map<String, LifecyclePolicyMetadata> policyMap = Collections.singletonMap(newPolicy.getName(),
+                new LifecyclePolicyMetadata(newPolicy, headers, randomNonNegativeLong(), randomNonNegativeLong()));
+        IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("new");
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
+            .put(IndexMetaData.builder("test")
+                .settings(Settings.builder()
+                    .put("index.uuid", "uuid")
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.version.created", Version.CURRENT.id)
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))
+            .build();
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            logger.info("--> metadata: {}", Strings.toString(builder));
+        }
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        // start with empty registry
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY, client);
+
+        // add new policy
+        registry.update(currentState);
+
+        assertThat(registry.getFirstStep(newPolicy.getName()), equalTo(policySteps.get(0)));
+        assertThat(registry.getLifecyclePolicyMap().size(), equalTo(1));
+        assertNotNull(registry.getLifecyclePolicyMap().get(newPolicy.getName()));
+        assertThat(registry.getLifecyclePolicyMap().get(newPolicy.getName()).getHeaders(), equalTo(headers));
+        assertThat(registry.getFirstStepMap().size(), equalTo(1));
+        assertThat(registry.getStepMap().size(), equalTo(1));
+        Map<Step.StepKey, Step> registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName());
+        assertThat(registeredStepsForPolicy.size(), equalTo(policySteps.size()));
+        for (Step step : policySteps) {
+            LifecycleExecutionState.Builder newIndexState = LifecycleExecutionState.builder();
+            newIndexState.setPhase(step.getKey().getPhase());
+            currentState = ClusterState.builder(currentState)
+                .metaData(MetaData.builder(currentState.metaData())
+                    .put(IndexMetaData.builder(currentState.metaData().index("test"))
+                        .settings(Settings.builder().put(currentState.metaData().index("test").getSettings()))
+                        .putCustom(ILM_CUSTOM_METADATA_KEY, newIndexState.build().asMap())))
+                .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+                .build();
+            registry.update(currentState);
+            assertThat(registeredStepsForPolicy.get(step.getKey()), equalTo(step));
+            assertThat(registry.getStep(metaData.index(index), step.getKey()), equalTo(step));
+        }
+
+        Map<String, LifecyclePolicyMetadata> registryPolicyMap = registry.getLifecyclePolicyMap();
+        Map<String, Step> registryFirstStepMap = registry.getFirstStepMap();
+        Map<String, Map<Step.StepKey, Step>> registryStepMap = registry.getStepMap();
+        registry.update(currentState);
+        assertThat(registry.getLifecyclePolicyMap(), equalTo(registryPolicyMap));
+        assertThat(registry.getFirstStepMap(), equalTo(registryFirstStepMap));
+        assertThat(registry.getStepMap(), equalTo(registryStepMap));
+
+        // remove policy
+        lifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), OperationMode.RUNNING);
+        currentState = ClusterState.builder(currentState)
+            .metaData(
+                MetaData.builder(metaData)
+                    .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)).build();
+        registry.update(currentState);
+        assertTrue(registry.getLifecyclePolicyMap().isEmpty());
+        assertTrue(registry.getFirstStepMap().isEmpty());
+        assertTrue(registry.getStepMap().isEmpty());
+    }
+
+    public void testUpdateChangedPolicy() {
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        String policyName = randomAlphaOfLengthBetween(5, 10);
+        LifecyclePolicy newPolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policyName);
+        Map<String, String> headers = new HashMap<>();
+        if (randomBoolean()) {
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        }
+        Map<String, LifecyclePolicyMetadata> policyMap = Collections.singletonMap(newPolicy.getName(),
+            new LifecyclePolicyMetadata(newPolicy, headers, randomNonNegativeLong(), randomNonNegativeLong()));
+        IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
+            .build();
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(NamedXContentRegistry.EMPTY, client);
+        // add new policy
+        registry.update(currentState);
+
+        // swap out policy
+        newPolicy = LifecyclePolicyTests.randomTestLifecyclePolicy(policyName);
+        lifecycleMetadata = new IndexLifecycleMetadata(Collections.singletonMap(policyName,
+            new LifecyclePolicyMetadata(newPolicy, Collections.emptyMap(),
+                randomNonNegativeLong(), randomNonNegativeLong())), OperationMode.RUNNING);
+        currentState = ClusterState.builder(currentState)
+            .metaData(MetaData.builder(metaData).putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)).build();
+        registry.update(currentState);
+        // TODO(talevy): assert changes... right now we do not support updates to policies. will require internal cleanup
+    }
+
+    public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Exception {
+        Index index = new Index("test", "uuid");
+        Client client = mock(Client.class);
+        Mockito.when(client.settings()).thenReturn(Settings.EMPTY);
+        String policyName = randomAlphaOfLength(5);
+        Map<String, LifecycleAction> actions = new HashMap<>();
+        actions.put("shrink", new ShrinkAction(1));
+        Map<String, Phase> phases = new HashMap<>();
+        Phase warmPhase = new Phase("warm", TimeValue.ZERO, actions);
+        PhaseExecutionInfo pei = new PhaseExecutionInfo(policyName, warmPhase, 1, randomNonNegativeLong());
+        String phaseJson = Strings.toString(pei);
+        phases.put("warm", new Phase("warm", TimeValue.ZERO, actions));
+        LifecyclePolicy newPolicy = new LifecyclePolicy(policyName, phases);
+        // Modify the policy
+        actions = new HashMap<>();
+        actions.put("shrink", new ShrinkAction(2));
+        phases = new HashMap<>();
+        phases.put("warm", new Phase("warm", TimeValue.ZERO, actions));
+        LifecyclePolicy updatedPolicy = new LifecyclePolicy(policyName, phases);
+        logger.info("--> policy: {}", newPolicy);
+        logger.info("--> updated policy: {}", updatedPolicy);
+        List<Step> policySteps = newPolicy.toSteps(client);
+        Map<String, String> headers = new HashMap<>();
+        if (randomBoolean()) {
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+            headers.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        }
+        Map<String, LifecyclePolicyMetadata> policyMap = Collections.singletonMap(newPolicy.getName(),
+            new LifecyclePolicyMetadata(newPolicy, headers, randomNonNegativeLong(), randomNonNegativeLong()));
+        IndexLifecycleMetadata lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase("warm");
+        lifecycleState.setPhaseDefinition(phaseJson);
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
+            .put(IndexMetaData.builder("test")
+                .settings(Settings.builder()
+                    .put("index.uuid", "uuid")
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.version.created", Version.CURRENT.id)
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policyName))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))
+            .build();
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            logger.info("--> metadata: {}", Strings.toString(builder));
+        }
+        String nodeId = randomAlphaOfLength(10);
+        DiscoveryNode masterNode = DiscoveryNode.createLocal(settings(Version.CURRENT)
+                .put(Node.NODE_MASTER_SETTING.getKey(), true).build(),
+            new TransportAddress(TransportAddress.META_ADDRESS, 9300), nodeId);
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        // start with empty registry
+        PolicyStepsRegistry registry = new PolicyStepsRegistry(REGISTRY, client);
+
+        // add new policy
+        registry.update(currentState);
+
+        Map<Step.StepKey, Step> registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName());
+        Step shrinkStep = registeredStepsForPolicy.entrySet().stream()
+            .filter(e -> e.getKey().getPhase().equals("warm") && e.getKey().getName().equals("shrink"))
+            .findFirst().get().getValue();
+        Step gotStep = registry.getStep(metaData.index(index), shrinkStep.getKey());
+        assertThat(((ShrinkStep) shrinkStep).getNumberOfShards(), equalTo(1));
+        assertThat(((ShrinkStep) gotStep).getNumberOfShards(), equalTo(1));
+
+        // Update the policy with the new policy, but keep the phase the same
+        policyMap = Collections.singletonMap(updatedPolicy.getName(), new LifecyclePolicyMetadata(updatedPolicy, headers,
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        lifecycleMetadata = new IndexLifecycleMetadata(policyMap, OperationMode.RUNNING);
+        metaData = MetaData.builder(metaData)
+            .putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata)
+            .build();
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            logger.info("--> metadata: {}", Strings.toString(builder));
+        }
+        currentState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+
+        // Update the policies
+        registry.update(currentState);
+
+        registeredStepsForPolicy = registry.getStepMap().get(newPolicy.getName());
+        shrinkStep = registeredStepsForPolicy.entrySet().stream()
+            .filter(e -> e.getKey().getPhase().equals("warm") && e.getKey().getName().equals("shrink"))
+            .findFirst().get().getValue();
+        gotStep = registry.getStep(metaData.index(index), shrinkStep.getKey());
+        assertThat(((ShrinkStep) shrinkStep).getNumberOfShards(), equalTo(2));
+        assertThat(((ShrinkStep) gotStep).getNumberOfShards(), equalTo(1));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/RandomStepInfo.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/RandomStepInfo.java
new file mode 100644
index 0000000000000..85084223481c3
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/RandomStepInfo.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class RandomStepInfo implements ToXContentObject {
+
+    private final String key;
+    private final String value;
+
+    public RandomStepInfo(Supplier<String> randomStringSupplier) {
+        this.key = randomStringSupplier.get();
+        this.value = randomStringSupplier.get();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(key, value);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, value);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        RandomStepInfo other = (RandomStepInfo) obj;
+        return Objects.equals(key, other.key) && Objects.equals(value, other.value);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTaskTests.java
new file mode 100644
index 0000000000000..a8b16d3ecfdf9
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/SetStepInfoUpdateTaskTests.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class SetStepInfoUpdateTaskTests extends ESTestCase {
+
+    String policy;
+    ClusterState clusterState;
+    Index index;
+
+    @Before
+    public void setupClusterState() {
+        policy = randomAlphaOfLength(10);
+        IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
+            .settings(settings(Version.CURRENT)
+                .put(LifecycleSettings.LIFECYCLE_NAME, policy))
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        index = indexMetadata.getIndex();
+        MetaData metaData = MetaData.builder()
+            .persistentSettings(settings(Version.CURRENT).build())
+            .put(IndexMetaData.builder(indexMetadata))
+            .build();
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+    }
+
+    public void testExecuteSuccessfullySet() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        ToXContentObject stepInfo = getRandomStepInfo();
+        setStateToKey(currentStepKey);
+
+        SetStepInfoUpdateTask task = new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo);
+        ClusterState newState = task.execute(clusterState);
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(newState.getMetaData().index(index));
+        StepKey actualKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
+        assertThat(actualKey, equalTo(currentStepKey));
+        assertThat(lifecycleState.getPhaseTime(), nullValue());
+        assertThat(lifecycleState.getActionTime(), nullValue());
+        assertThat(lifecycleState.getStepTime(), nullValue());
+
+        XContentBuilder infoXContentBuilder = JsonXContent.contentBuilder();
+        stepInfo.toXContent(infoXContentBuilder, ToXContent.EMPTY_PARAMS);
+        String expectedCauseValue = BytesReference.bytes(infoXContentBuilder).utf8ToString();
+        assertThat(lifecycleState.getStepInfo(), equalTo(expectedCauseValue));
+    }
+
+    private ToXContentObject getRandomStepInfo() {
+        String key = randomAlphaOfLength(20);
+        String value = randomAlphaOfLength(20);
+        return (b, p) -> {
+            b.startObject();
+            b.field(key, value);
+            b.endObject();
+            return b;
+        };
+    }
+
+    public void testExecuteNoopDifferentStep() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        StepKey notCurrentStepKey = new StepKey("not-current", "not-current", "not-current");
+        ToXContentObject stepInfo = getRandomStepInfo();
+        setStateToKey(notCurrentStepKey);
+        SetStepInfoUpdateTask task = new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo);
+        ClusterState newState = task.execute(clusterState);
+        assertThat(newState, sameInstance(clusterState));
+    }
+
+    public void testExecuteNoopDifferentPolicy() throws IOException {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        ToXContentObject stepInfo = getRandomStepInfo();
+        setStateToKey(currentStepKey);
+        setStatePolicy("not-" + policy);
+        SetStepInfoUpdateTask task = new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo);
+        ClusterState newState = task.execute(clusterState);
+        assertThat(newState, sameInstance(clusterState));
+    }
+
+    public void testOnFailure() {
+        StepKey currentStepKey = new StepKey("current-phase", "current-action", "current-name");
+        ToXContentObject stepInfo = getRandomStepInfo();
+
+        setStateToKey(currentStepKey);
+
+        SetStepInfoUpdateTask task = new SetStepInfoUpdateTask(index, policy, currentStepKey, stepInfo);
+        Exception expectedException = new RuntimeException();
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> task.onFailure(randomAlphaOfLength(10), expectedException));
+        assertEquals("policy [" + policy + "] for index [" + index.getName() + "] failed trying to set step info for step ["
+                + currentStepKey + "].", exception.getMessage());
+        assertSame(expectedException, exception.getCause());
+    }
+
+    private void setStatePolicy(String policy) {
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.metaData())
+                .updateSettings(Settings.builder()
+                    .put(LifecycleSettings.LIFECYCLE_NAME, policy).build(), index.getName())).build();
+
+    }
+    private void setStateToKey(StepKey stepKey) {
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(
+            LifecycleExecutionState.fromIndexMetadata(clusterState.metaData().index(index)));
+        lifecycleState.setPhase(stepKey.getPhase());
+        lifecycleState.setAction(stepKey.getAction());
+        lifecycleState.setStep(stepKey.getName());
+
+        clusterState = ClusterState.builder(clusterState)
+            .metaData(MetaData.builder(clusterState.getMetaData())
+                .put(IndexMetaData.builder(clusterState.getMetaData().index(index))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap()))).build();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeValueScheduleTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeValueScheduleTests.java
new file mode 100644
index 0000000000000..f34a2aa458d16
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeValueScheduleTests.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.concurrent.TimeUnit;
+
+public class TimeValueScheduleTests extends ESTestCase {
+
+    public TimeValueSchedule createRandomInstance() {
+        return new TimeValueSchedule(createRandomTimeValue());
+    }
+
+    private TimeValue createRandomTimeValue() {
+        return new TimeValue(randomLongBetween(1, 10000), randomFrom(TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS));
+    }
+
+    public void testHascodeAndEquals() {
+        for (int i = 0; i < 20; i++) {
+            EqualsHashCodeTestUtils.checkEqualsAndHashCode(createRandomInstance(),
+                    instance -> new TimeValueSchedule(instance.getInterval()),
+                    instance -> new TimeValueSchedule(randomValueOtherThan(instance.getInterval(), () -> createRandomTimeValue())));
+        }
+    }
+
+    public void testNextScheduledTimeFirstTriggerNotReached() {
+        long start = randomNonNegativeLong();
+        TimeValue interval = createRandomTimeValue();
+        long triggerTime = start + interval.millis();
+        long now = start + randomLongBetween(0, interval.millis() - 1);
+        TimeValueSchedule schedule = new TimeValueSchedule(interval);
+        assertEquals(triggerTime, schedule.nextScheduledTimeAfter(start, now));
+    }
+
+    public void testNextScheduledTimeAtFirstInterval() {
+        long start = randomNonNegativeLong();
+        TimeValue interval = createRandomTimeValue();
+        long triggerTime = start + 2 * interval.millis();
+        long now = start + interval.millis();
+        TimeValueSchedule schedule = new TimeValueSchedule(interval);
+        assertEquals(triggerTime, schedule.nextScheduledTimeAfter(start, now));
+    }
+
+    public void testNextScheduledTimeAtStartTime() {
+        long start = randomNonNegativeLong();
+        TimeValue interval = createRandomTimeValue();
+        long triggerTime = start + interval.millis();
+        TimeValueSchedule schedule = new TimeValueSchedule(interval);
+        assertEquals(triggerTime, schedule.nextScheduledTimeAfter(start, start));
+    }
+
+    public void testNextScheduledTimeAfterFirstTrigger() {
+        long start = randomNonNegativeLong();
+        TimeValue interval = createRandomTimeValue();
+        long numberIntervalsPassed = randomLongBetween(0, 10000);
+        long triggerTime = start + (numberIntervalsPassed + 1) * interval.millis();
+        long now = start
+                + randomLongBetween(numberIntervalsPassed * interval.millis(), (numberIntervalsPassed + 1) * interval.millis() - 1);
+        TimeValueSchedule schedule = new TimeValueSchedule(interval);
+        assertEquals(triggerTime, schedule.nextScheduledTimeAfter(start, now));
+    }
+
+    public void testInvalidInterval() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new TimeValueSchedule(new TimeValue(0)));
+        assertEquals("interval must be greater than 0 milliseconds", exception.getMessage());
+    }
+}
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index eddedda59b3e8..ec4b96cb52bea 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.logstash;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -59,7 +59,7 @@ public Collection<Module> createGuiceModules() {
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
             TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_NAME + ".json", templates, LOGSTASH_TEMPLATE_NAME,
-                    Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, Loggers.getLogger(Logstash.class));
+                    Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, LogManager.getLogger(Logstash.class));
             return templates;
         };
     }
diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
index cc5a2cd68dde5..b47016c134459 100644
--- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
+++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
@@ -7,6 +7,7 @@ dependencies {
 }
 
 integTestCluster {
+    setting 'xpack.ilm.enabled', 'false'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.watcher.enabled', 'false'
diff --git a/x-pack/plugin/ml/qa/build.gradle b/x-pack/plugin/ml/qa/build.gradle
index 5b3dcd7c8508a..35bd236df5c23 100644
--- a/x-pack/plugin/ml/qa/build.gradle
+++ b/x-pack/plugin/ml/qa/build.gradle
@@ -15,16 +15,3 @@ subprojects {
         }
     }
 }
-
-gradle.projectsEvaluated {
-    subprojects {
-        Task assemble = project.tasks.findByName('assemble')
-        if (assemble) {
-            assemble.enabled = false
-        }
-        Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
-        if (dependenciesInfo) {
-            dependenciesInfo.enabled = false
-        }
-    }
-}
diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle
index a24036651d504..2aa5d47acef0d 100644
--- a/x-pack/plugin/ml/qa/disabled/build.gradle
+++ b/x-pack/plugin/ml/qa/disabled/build.gradle
@@ -7,6 +7,7 @@ dependencies {
 }
 
 integTestCluster {
+    setting 'xpack.ilm.enabled', 'false'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.ml.enabled', 'false'
     numNodes = 1
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
index 33a611fcb57e2..9c19ffe639f7d 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.ml.integration;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -204,7 +204,7 @@ public void testCategorizationPerformance() {
         flushJob(jobId, false);
 
         long duration = System.currentTimeMillis() - startTime;
-        Loggers.getLogger(CategorizationIT.class).info("Performance test with tokenization in " +
+        LogManager.getLogger(CategorizationIT.class).info("Performance test with tokenization in " +
                 (MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA ? "Java" : "C++") + " took " + duration + "ms");
     }
 
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
index 5155f5446b2f9..b64d09ef8672e 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
 import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
+import org.elasticsearch.xpack.core.rollup.job.RollupJob;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.junit.After;
 import org.junit.Before;
@@ -27,6 +28,7 @@
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -63,6 +65,16 @@ private void setupDataAccessRole(String index) throws IOException {
         client().performRequest(request);
     }
 
+    private void setupFullAccessRole(String index) throws IOException {
+        Request request = new Request("PUT", "/_xpack/security/role/test_data_access");
+        request.setJsonEntity("{"
+            + "  \"indices\" : ["
+            + "    { \"names\": [\"" + index + "\"], \"privileges\": [\"all\"] }"
+            + "  ]"
+            + "}");
+        client().performRequest(request);
+    }
+
     private void setupUser(String user, List<String> roles) throws IOException {
         String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());
 
@@ -359,7 +371,75 @@ public void testInsufficientSearchPrivilegesOnPut() throws Exception {
 
         assertThat(e.getMessage(), containsString("Cannot create datafeed"));
         assertThat(e.getMessage(),
-                containsString("user ml_admin lacks permissions on the indices to be searched"));
+                containsString("user ml_admin lacks permissions on the indices"));
+    }
+
+    public void testInsufficientSearchPrivilegesOnPutWithRollup() throws Exception {
+        setupDataAccessRole("airline-data-aggs-rollup");
+        String jobId = "privs-put-job-rollup";
+        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+        createJobRequest.setJsonEntity("{\n"
+            + "  \"description\": \"Aggs job\",\n"
+            + "  \"analysis_config\": {\n"
+            + "    \"bucket_span\": \"1h\",\n"
+            + "    \"summary_count_field_name\": \"doc_count\",\n"
+            + "    \"detectors\": [\n"
+            + "      {\n"
+            + "        \"function\": \"mean\",\n"
+            + "        \"field_name\": \"responsetime\",\n"
+            + "        \"by_field_name\": \"airline\"\n"
+            + "      }\n"
+            + "    ]\n"
+            + "  },\n"
+            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
+            + "}");
+        client().performRequest(createJobRequest);
+
+        String rollupJobId = "rollup-" + jobId;
+        Request createRollupRequest = new Request("PUT", "/_xpack/rollup/job/" + rollupJobId);
+        createRollupRequest.setJsonEntity("{\n"
+            + "\"index_pattern\": \"airline-data-aggs\",\n"
+            + "    \"rollup_index\": \"airline-data-aggs-rollup\",\n"
+            + "    \"cron\": \"*/30 * * * * ?\",\n"
+            + "    \"page_size\" :1000,\n"
+            + "    \"groups\" : {\n"
+            + "      \"date_histogram\": {\n"
+            + "        \"field\": \"time stamp\",\n"
+            + "        \"interval\": \"2m\",\n"
+            + "        \"delay\": \"7d\"\n"
+            + "      },\n"
+            + "      \"terms\": {\n"
+            + "        \"fields\": [\"airline\"]\n"
+            + "      }"
+            + "    },\n"
+            + "    \"metrics\": [\n"
+            + "        {\n"
+            + "            \"field\": \"responsetime\",\n"
+            + "            \"metrics\": [\"avg\",\"min\",\"max\",\"sum\"]\n"
+            + "        },\n"
+            + "        {\n"
+            + "            \"field\": \"time stamp\",\n"
+            + "            \"metrics\": [\"min\",\"max\"]\n"
+            + "        }\n"
+            + "    ]\n"
+            + "}");
+        client().performRequest(createRollupRequest);
+
+        String datafeedId = "datafeed-" + jobId;
+        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":3600000},"
+            + "\"aggregations\":{"
+            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+            + "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";
+
+
+        ResponseException e = expectThrows(ResponseException.class, () ->
+            new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "doc")
+                .setAggregations(aggregations)
+                .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS) //want to search, but no admin access
+                .build());
+        assertThat(e.getMessage(), containsString("Cannot create datafeed"));
+        assertThat(e.getMessage(),
+            containsString("user ml_admin_plus_data lacks permissions on the indices"));
     }
 
     public void testInsufficientSearchPrivilegesOnPreview() throws Exception {
@@ -615,7 +695,7 @@ public void testLookbackWithoutPermissions() throws Exception {
         // There should be a notification saying that there was a problem extracting data
         client().performRequest(new Request("POST", "/_refresh"));
         Response notificationsResponse = client().performRequest(
-                new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?q=job_id:" + jobId));
+                new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
         String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
         assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
                 "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\""));
@@ -663,6 +743,171 @@ public void testLookbackWithPipelineBucketAgg() throws Exception {
         assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
     }
 
+    public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throws Exception {
+        String jobId = "aggs-histogram-rollup-job";
+        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+        createJobRequest.setJsonEntity("{\n"
+            + "  \"description\": \"Aggs job\",\n"
+            + "  \"analysis_config\": {\n"
+            + "    \"bucket_span\": \"1h\",\n"
+            + "    \"summary_count_field_name\": \"doc_count\",\n"
+            + "    \"detectors\": [\n"
+            + "      {\n"
+            + "        \"function\": \"mean\",\n"
+            + "        \"field_name\": \"responsetime\",\n"
+            + "        \"by_field_name\": \"airline\"\n"
+            + "      }\n"
+            + "    ]\n"
+            + "  },\n"
+            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
+            + "}");
+        client().performRequest(createJobRequest);
+
+        String rollupJobId = "rollup-" + jobId;
+        Request createRollupRequest = new Request("PUT", "/_xpack/rollup/job/" + rollupJobId);
+        createRollupRequest.setJsonEntity("{\n"
+            + "\"index_pattern\": \"airline-data-aggs\",\n"
+            + "    \"rollup_index\": \"airline-data-aggs-rollup\",\n"
+            + "    \"cron\": \"*/30 * * * * ?\",\n"
+            + "    \"page_size\" :1000,\n"
+            + "    \"groups\" : {\n"
+            + "      \"date_histogram\": {\n"
+            + "        \"field\": \"time stamp\",\n"
+            + "        \"interval\": \"2m\",\n"
+            + "        \"delay\": \"7d\"\n"
+            + "      },\n"
+            + "      \"terms\": {\n"
+            + "        \"fields\": [\"airline\"]\n"
+            + "      }"
+            + "    },\n"
+            + "    \"metrics\": [\n"
+            + "        {\n"
+            + "            \"field\": \"responsetime\",\n"
+            + "            \"metrics\": [\"avg\",\"min\",\"max\",\"sum\"]\n"
+            + "        },\n"
+            + "        {\n"
+            + "            \"field\": \"time stamp\",\n"
+            + "            \"metrics\": [\"min\",\"max\"]\n"
+            + "        }\n"
+            + "    ]\n"
+            + "}");
+        client().performRequest(createRollupRequest);
+        client().performRequest(new Request("POST", "/_xpack/rollup/job/" + rollupJobId + "/_start"));
+
+        assertBusy(() -> {
+            Response getRollup = client().performRequest(new Request("GET", "/_xpack/rollup/job/" + rollupJobId));
+            String body = EntityUtils.toString(getRollup.getEntity());
+            assertThat(body, containsString("\"job_state\":\"started\""));
+            assertThat(body, containsString("\"rollups_indexed\":4"));
+        }, 60, TimeUnit.SECONDS);
+
+        client().performRequest(new Request("POST", "/_xpack/rollup/job/" + rollupJobId + "/_stop"));
+        assertBusy(() -> {
+            Response getRollup = client().performRequest(new Request("GET", "/_xpack/rollup/job/" + rollupJobId));
+            assertThat(EntityUtils.toString(getRollup.getEntity()), containsString("\"job_state\":\"stopped\""));
+        }, 60, TimeUnit.SECONDS);
+
+        final Request refreshRollupIndex = new Request("POST", "airline-data-aggs-rollup/_refresh");
+        client().performRequest(refreshRollupIndex);
+
+        String datafeedId = "datafeed-" + jobId;
+        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":3600000},"
+            + "\"aggregations\":{"
+            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+            + "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";
+        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "response").setAggregations(aggregations).build();
+        openJob(client(), jobId);
+
+        startDatafeedAndWaitUntilStopped(datafeedId);
+        waitUntilJobIsClosed(jobId);
+        Response jobStatsResponse = client().performRequest(new Request("GET",
+            MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
+        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
+        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
+        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
+    }
+
+    public void testLookbackWithoutPermissionsAndRollup() throws Exception {
+        setupFullAccessRole("airline-data-aggs-rollup");
+        String jobId = "rollup-permission-test-network-job";
+        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+        createJobRequest.setJsonEntity("{\n"
+            + "  \"description\": \"Aggs job\",\n"
+            + "  \"analysis_config\": {\n"
+            + "    \"bucket_span\": \"1h\",\n"
+            + "    \"summary_count_field_name\": \"doc_count\",\n"
+            + "    \"detectors\": [\n"
+            + "      {\n"
+            + "        \"function\": \"mean\",\n"
+            + "        \"field_name\": \"responsetime\",\n"
+            + "        \"by_field_name\": \"airline\"\n"
+            + "      }\n"
+            + "    ]\n"
+            + "  },\n"
+            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
+            + "}");
+        client().performRequest(createJobRequest);
+
+        String rollupJobId = "rollup-" + jobId;
+        Request createRollupRequest = new Request("PUT", "/_xpack/rollup/job/" + rollupJobId);
+        createRollupRequest.setJsonEntity("{\n"
+            + "\"index_pattern\": \"airline-data-aggs\",\n"
+            + "    \"rollup_index\": \"airline-data-aggs-rollup\",\n"
+            + "    \"cron\": \"*/30 * * * * ?\",\n"
+            + "    \"page_size\" :1000,\n"
+            + "    \"groups\" : {\n"
+            + "      \"date_histogram\": {\n"
+            + "        \"field\": \"time stamp\",\n"
+            + "        \"interval\": \"2m\",\n"
+            + "        \"delay\": \"7d\"\n"
+            + "      },\n"
+            + "      \"terms\": {\n"
+            + "        \"fields\": [\"airline\"]\n"
+            + "      }"
+            + "    },\n"
+            + "    \"metrics\": [\n"
+            + "        {\n"
+            + "            \"field\": \"responsetime\",\n"
+            + "            \"metrics\": [\"avg\",\"min\",\"max\",\"sum\"]\n"
+            + "        },\n"
+            + "        {\n"
+            + "            \"field\": \"time stamp\",\n"
+            + "            \"metrics\": [\"min\",\"max\"]\n"
+            + "        }\n"
+            + "    ]\n"
+            + "}");
+        client().performRequest(createRollupRequest);
+
+        String datafeedId = "datafeed-" + jobId;
+        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":3600000},"
+            + "\"aggregations\":{"
+            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
+            + "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";
+
+
+        // At the time we create the datafeed the user can access the network-data index that we have access to
+        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup", "doc")
+            .setAggregations(aggregations)
+            .setChunkingTimespan("300s")
+            .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
+            .build();
+
+        // Change the role so that the user can no longer access network-data
+        setupFullAccessRole("some-other-data");
+
+        openJob(client(), jobId);
+
+        startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS);
+        waitUntilJobIsClosed(jobId);
+        // There should be a notification saying that there was a problem extracting data
+        client().performRequest(new Request("POST", "/_refresh"));
+        Response notificationsResponse = client().performRequest(
+            new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
+        String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
+        assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
+            "action [indices:admin/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data]\""));
+    }
+
     public void testRealtime() throws Exception {
         String jobId = "job-realtime-1";
         createJob(jobId, "airline");
@@ -882,7 +1127,8 @@ public static void openJob(RestClient client, String jobId) throws IOException {
     @After
     public void clearMlState() throws Exception {
         new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata();
-        ESRestTestCase.waitForPendingTasks(adminClient());
+        // Don't check rollup jobs because we clear them in the superclass.
+        waitForPendingTasks(adminClient(), taskName -> taskName.startsWith(RollupJob.NAME));
     }
 
     private static class DatafeedBuilder {
diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
index 88ca4dd118ea4..f856c3d4c5ff4 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
@@ -7,6 +7,7 @@ dependencies {
 }
 
 integTestCluster {
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/InvalidLicenseEnforcer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/InvalidLicenseEnforcer.java
index 7c9f163a2250d..21e06362a70a3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/InvalidLicenseEnforcer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/InvalidLicenseEnforcer.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.ml;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -20,9 +19,8 @@ public class InvalidLicenseEnforcer extends AbstractComponent {
     private final DatafeedManager datafeedManager;
     private final AutodetectProcessManager autodetectProcessManager;
 
-    InvalidLicenseEnforcer(Settings settings, XPackLicenseState licenseState, ThreadPool threadPool,
+    InvalidLicenseEnforcer(XPackLicenseState licenseState, ThreadPool threadPool,
                            DatafeedManager datafeedManager, AutodetectProcessManager autodetectProcessManager) {
-        super(settings);
         this.threadPool = threadPool;
         this.licenseState = licenseState;
         this.datafeedManager = datafeedManager;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index e0c44faec7471..77f898dcb8773 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
@@ -22,7 +23,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -267,7 +267,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
     public static final Setting<Integer> MAX_LAZY_ML_NODES =
         Setting.intSetting("xpack.ml.max_lazy_ml_nodes", 0, 0, 3, Property.Dynamic, Property.NodeScope);
 
-    private static final Logger logger = Loggers.getLogger(XPackPlugin.class);
+    private static final Logger logger = LogManager.getLogger(XPackPlugin.class);
 
     private final Settings settings;
     private final Environment env;
@@ -372,13 +372,13 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         Auditor auditor = new Auditor(client, clusterService.getNodeName());
         JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings);
-        JobConfigProvider jobConfigProvider = new JobConfigProvider(client, settings);
-        DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
-        UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(settings, client, clusterService, threadPool);
+        JobConfigProvider jobConfigProvider = new JobConfigProvider(client);
+        DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
+        UpdateJobProcessNotifier notifier = new UpdateJobProcessNotifier(client, clusterService, threadPool);
         JobManager jobManager = new JobManager(env, settings, jobResultsProvider, clusterService, auditor, threadPool, client, notifier);
 
-        JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(settings, client);
-        JobResultsPersister jobResultsPersister = new JobResultsPersister(settings, client);
+        JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(client);
+        JobResultsPersister jobResultsPersister = new JobResultsPersister(client);
 
         AutodetectProcessFactory autodetectProcessFactory;
         NormalizerProcessFactory normalizerProcessFactory;
@@ -422,7 +422,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                 autodetectProcessManager);
 
         // This object's constructor attaches to the license state, so there's no need to retain another reference to it
-        new InvalidLicenseEnforcer(settings, getLicenseState(), threadPool, datafeedManager, autodetectProcessManager);
+        new InvalidLicenseEnforcer(getLicenseState(), threadPool, datafeedManager, autodetectProcessManager);
 
         // run node startup tasks
         autodetectProcessManager.onNodeStartup();
@@ -434,11 +434,11 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                 datafeedConfigProvider,
                 jobManager,
                 autodetectProcessManager,
-                new MlInitializationService(settings, threadPool, clusterService, client),
+                new MlInitializationService(threadPool, clusterService, client),
                 jobDataCountsPersister,
                 datafeedManager,
                 auditor,
-                new MlAssignmentNotifier(settings, auditor, clusterService)
+                new MlAssignmentNotifier(auditor, clusterService)
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
index 329d926cc078b..9eec0d848f072 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.ElasticsearchException;
@@ -16,7 +17,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.Platforms;
@@ -84,7 +84,7 @@ public MachineLearningFeatureSet(Environment environment, ClusterService cluster
                     }
                 }
             } catch (IOException | TimeoutException e) {
-                Loggers.getLogger(MachineLearningFeatureSet.class).error("Cannot get native code info for Machine Learning", e);
+                LogManager.getLogger(MachineLearningFeatureSet.class).error("Cannot get native code info for Machine Learning", e);
                 throw new ElasticsearchException("Cannot communicate with Machine Learning native code");
             }
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java
index c7e867bb7de70..67486ba75792a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlAssignmentNotifier.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.Assignment;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
@@ -24,15 +24,15 @@
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-public class MlAssignmentNotifier extends AbstractComponent implements ClusterStateListener, LocalNodeMasterListener {
+public class MlAssignmentNotifier implements ClusterStateListener, LocalNodeMasterListener {
+    private static final Logger logger = LogManager.getLogger(MlAssignmentNotifier.class);
 
     private final Auditor auditor;
     private final ClusterService clusterService;
 
     private final AtomicBoolean enabled = new AtomicBoolean(false);
 
-    MlAssignmentNotifier(Settings settings, Auditor auditor, ClusterService clusterService) {
-        super(settings);
+    MlAssignmentNotifier(Auditor auditor, ClusterService clusterService) {
         this.auditor = auditor;
         this.clusterService = clusterService;
         clusterService.addLocalNodeMasterListener(this);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java
index 4d0911f6691e1..190933b1e9316 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
@@ -32,7 +32,7 @@
  */
 public class MlDailyMaintenanceService implements Releasable {
 
-    private static final Logger LOGGER = Loggers.getLogger(MlDailyMaintenanceService.class);
+    private static final Logger LOGGER = LogManager.getLogger(MlDailyMaintenanceService.class);
 
     private static final int MAX_TIME_OFFSET_MINUTES = 120;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
index c96a12ffa1047..016fcd5e9287b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -23,8 +22,7 @@ class MlInitializationService extends AbstractComponent implements ClusterStateL
 
     private volatile MlDailyMaintenanceService mlDailyMaintenanceService;
 
-    MlInitializationService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client) {
-        super(settings);
+    MlInitializationService(ThreadPool threadPool, ClusterService clusterService, Client client) {
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.client = client;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
index efc0517900ec4..261534556968a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
@@ -28,7 +28,6 @@ public MlLifeCycleService(Environment environment, ClusterService clusterService
 
     public MlLifeCycleService(Environment environment, ClusterService clusterService, DatafeedManager datafeedManager,
                               AutodetectProcessManager autodetectProcessManager) {
-        super(environment.settings());
         this.environment = environment;
         this.datafeedManager = datafeedManager;
         this.autodetectProcessManager = autodetectProcessManager;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
index f441e42acc517..3178c9980b65e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
@@ -50,7 +50,7 @@ public TransportDeleteDatafeedAction(Settings settings, TransportService transpo
         super(settings, DeleteDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, DeleteDatafeedAction.Request::new);
         this.client = client;
-        this.datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
+        this.datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
         this.persistentTasksService = persistentTasksService;
         this.clusterService = clusterService;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
index e1d2e44c8d04e..a848634b30e90 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
@@ -47,7 +47,7 @@ public TransportGetDatafeedsAction(Settings settings, TransportService transport
         super(settings, GetDatafeedsAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, GetDatafeedsAction.Request::new);
 
-        datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
+        datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index edd61ec595123..5b4be78596a64 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.action;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
@@ -33,7 +34,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -658,7 +658,7 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut
 
         public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterService,
                                               AutodetectProcessManager autodetectProcessManager) {
-            super(settings, MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
+            super(MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
             this.autodetectProcessManager = autodetectProcessManager;
             this.fallbackMaxNumberOfOpenJobs = AutodetectProcessManager.MAX_OPEN_JOBS_PER_NODE.get(settings);
             this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
@@ -757,7 +757,7 @@ void setMaxLazyMLNodes(int maxLazyMLNodes) {
 
     public static class JobTask extends AllocatedPersistentTask implements OpenJobAction.JobTaskMatcher {
 
-        private static final Logger LOGGER = Loggers.getLogger(JobTask.class);
+        private static final Logger LOGGER = LogManager.getLogger(JobTask.class);
 
         private final String jobId;
         private volatile AutodetectProcessManager autodetectProcessManager;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
index 874ee8f71f5ad..62eb4e5039568 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
@@ -34,6 +35,8 @@
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
@@ -47,6 +50,9 @@
 import java.util.Collections;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
 public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDatafeedAction.Request, PutDatafeedAction.Response> {
 
     private final XPackLicenseState licenseState;
@@ -69,8 +75,8 @@ public TransportPutDatafeedAction(Settings settings, TransportService transportS
         this.clusterService = clusterService;
         this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
                 new SecurityContext(settings, threadPool.getThreadContext()) : null;
-        this.datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
-        this.jobConfigProvider = new JobConfigProvider(client, settings);
+        this.datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
+        this.jobConfigProvider = new JobConfigProvider(client);
     }
 
     @Override
@@ -89,23 +95,48 @@ protected void masterOperation(PutDatafeedAction.Request request, ClusterState s
         // If security is enabled only create the datafeed if the user requesting creation has
         // permission to read the indices the datafeed is going to read from
         if (licenseState.isAuthAllowed()) {
-            final String username = securityContext.getUser().principal();
-            ActionListener<HasPrivilegesResponse> privResponseListener = ActionListener.wrap(
-                    r -> handlePrivsResponse(username, request, r, listener),
-                    listener::onFailure);
 
-            HasPrivilegesRequest privRequest = new HasPrivilegesRequest();
+            final String[] indices = request.getDatafeed().getIndices().toArray(new String[0]);
+
+            final String username = securityContext.getUser().principal();
+            final HasPrivilegesRequest privRequest = new HasPrivilegesRequest();
+            privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
             privRequest.username(username);
             privRequest.clusterPrivileges(Strings.EMPTY_ARRAY);
-            // We just check for permission to use the search action.  In reality we'll also
-            // use the scroll action, but that's considered an implementation detail.
-            privRequest.indexPrivileges(RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(request.getDatafeed().getIndices().toArray(new String[0]))
-                    .privileges(SearchAction.NAME)
-                    .build());
-            privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
 
-            client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+            final RoleDescriptor.IndicesPrivileges.Builder indicesPrivilegesBuilder = RoleDescriptor.IndicesPrivileges.builder()
+                .indices(indices);
+
+            ActionListener<HasPrivilegesResponse> privResponseListener = ActionListener.wrap(
+                r -> handlePrivsResponse(username, request, r, listener),
+                listener::onFailure);
+
+            ActionListener<GetRollupIndexCapsAction.Response> getRollupIndexCapsActionHandler = ActionListener.wrap(
+                response -> {
+                    if (response.getJobs().isEmpty()) { // This means no rollup indexes are in the config
+                        indicesPrivilegesBuilder.privileges(SearchAction.NAME);
+                    } else {
+                        indicesPrivilegesBuilder.privileges(SearchAction.NAME, RollupSearchAction.NAME);
+                    }
+                    privRequest.indexPrivileges(indicesPrivilegesBuilder.build());
+                    client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+                },
+                e -> {
+                    if (e instanceof IndexNotFoundException) {
+                        indicesPrivilegesBuilder.privileges(SearchAction.NAME);
+                        privRequest.indexPrivileges(indicesPrivilegesBuilder.build());
+                        client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+                    } else {
+                        listener.onFailure(e);
+                    }
+                }
+            );
+
+            executeAsyncWithOrigin(client,
+                ML_ORIGIN,
+                GetRollupIndexCapsAction.INSTANCE,
+                new GetRollupIndexCapsAction.Request(indices),
+                getRollupIndexCapsActionHandler);
         } else {
             putDatafeed(request, threadPool.getThreadContext().getHeaders(), listener);
         }
@@ -126,8 +157,7 @@ private void handlePrivsResponse(String username, PutDatafeedAction.Request requ
             builder.endObject();
 
             listener.onFailure(Exceptions.authorizationError("Cannot create datafeed [{}]" +
-                            " because user {} lacks permissions on the indices to be" +
-                            " searched: {}",
+                            " because user {} lacks permissions on the indices: {}",
                     request.getDatafeed().getId(), username, Strings.toString(builder)));
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index aa416b6fa1d54..95103216b8691 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -302,7 +302,7 @@ public static class StartDatafeedPersistentTasksExecutor extends PersistentTasks
         private final IndexNameExpressionResolver resolver;
 
         public StartDatafeedPersistentTasksExecutor(Settings settings, DatafeedManager datafeedManager) {
-            super(settings, MlTasks.DATAFEED_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
+            super(MlTasks.DATAFEED_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
             this.datafeedManager = datafeedManager;
             this.resolver = new IndexNameExpressionResolver(settings);
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
index 3fd90b5d21c19..904cc1ec7a321 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
@@ -18,13 +18,13 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
-import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
@@ -46,8 +46,8 @@ public TransportUpdateDatafeedAction(Settings settings, TransportService transpo
         super(settings, UpdateDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, UpdateDatafeedAction.Request::new);
 
-        datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
-        jobConfigProvider = new JobConfigProvider(client, settings);
+        datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
+        jobConfigProvider = new JobConfigProvider(client);
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
index 20900c3f0d8e7..127bd20e599dd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.ml.datafeed;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -20,10 +20,10 @@
 import org.elasticsearch.xpack.core.ml.action.PersistJobAction;
 import org.elasticsearch.xpack.core.ml.action.PostDataAction;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
-import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
+import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
 import java.io.ByteArrayOutputStream;
@@ -39,7 +39,7 @@
 
 class DatafeedJob {
 
-    private static final Logger LOGGER = Loggers.getLogger(DatafeedJob.class);
+    private static final Logger LOGGER = LogManager.getLogger(DatafeedJob.class);
     private static final int NEXT_TASK_DELAY_MS = 100;
 
     private final Auditor auditor;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
index 2f3c93be92b3f..f83775e51565f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobBuilder.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedJobValidator;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
-import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
+import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
+import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
@@ -52,8 +52,8 @@ public DatafeedJobBuilder(Client client, Settings settings, NamedXContentRegistr
     void build(String datafeedId, ActionListener<DatafeedJob> listener) {
 
         JobResultsProvider jobResultsProvider = new JobResultsProvider(client, settings);
-        JobConfigProvider jobConfigProvider = new JobConfigProvider(client, settings);
-        DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, settings, xContentRegistry);
+        JobConfigProvider jobConfigProvider = new JobConfigProvider(client);
+        DatafeedConfigProvider datafeedConfigProvider = new DatafeedConfigProvider(client, xContentRegistry);
 
         build(datafeedId, jobResultsProvider, jobConfigProvider, datafeedConfigProvider, listener);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
index c801118fe0fa7..d49367de429f9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
@@ -64,7 +63,6 @@ public class DatafeedManager extends AbstractComponent {
 
     public DatafeedManager(ThreadPool threadPool, Client client, ClusterService clusterService, DatafeedJobBuilder datafeedJobBuilder,
                            Supplier<Long> currentTimeSupplier, Auditor auditor) {
-        super(Settings.EMPTY);
         this.client = Objects.requireNonNull(client);
         this.clusterService = Objects.requireNonNull(clusterService);
         this.threadPool = threadPool;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
index a45bfc7822071..b1ee1776bfe25 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.ml.datafeed;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -24,7 +24,7 @@
 
 public class DatafeedNodeSelector {
 
-    private static final Logger LOGGER = Loggers.getLogger(DatafeedNodeSelector.class);
+    private static final Logger LOGGER = LogManager.getLogger(DatafeedNodeSelector.class);
 
     private final String datafeedId;
     private final String jobId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java
index 8fd1ced17293a..77e2c695db7d5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java
@@ -5,14 +5,19 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
 import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationDataExtractorFactory;
 import org.elasticsearch.xpack.ml.datafeed.extractor.chunked.ChunkedDataExtractorFactory;
+import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.RollupDataExtractorFactory;
 import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorFactory;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
 
 public interface DataExtractorFactory {
     DataExtractor newExtractor(long start, long end);
@@ -22,16 +27,44 @@ public interface DataExtractorFactory {
      */
     static void create(Client client, DatafeedConfig datafeed, Job job, ActionListener<DataExtractorFactory> listener) {
         ActionListener<DataExtractorFactory> factoryHandler = ActionListener.wrap(
-                factory -> listener.onResponse(datafeed.getChunkingConfig().isEnabled()
-                        ? new ChunkedDataExtractorFactory(client, datafeed, job, factory) : factory)
-                , listener::onFailure
+            factory -> listener.onResponse(datafeed.getChunkingConfig().isEnabled()
+                ? new ChunkedDataExtractorFactory(client, datafeed, job, factory) : factory)
+            , listener::onFailure
         );
 
-        boolean isScrollSearch = datafeed.hasAggregations() == false;
-        if (isScrollSearch) {
-            ScrollDataExtractorFactory.create(client, datafeed, job, factoryHandler);
-        } else {
-            factoryHandler.onResponse(new AggregationDataExtractorFactory(client, datafeed, job));
-        }
+        ActionListener<GetRollupIndexCapsAction.Response> getRollupIndexCapsActionHandler = ActionListener.wrap(
+            response -> {
+                if (response.getJobs().isEmpty()) { // This means no rollup indexes are in the config
+                    if (datafeed.hasAggregations()) {
+                        factoryHandler.onResponse(new AggregationDataExtractorFactory(client, datafeed, job));
+                    } else {
+                        ScrollDataExtractorFactory.create(client, datafeed, job, factoryHandler);
+                    }
+                } else {
+                    if (datafeed.hasAggregations()) { // Rollup indexes require aggregations
+                        RollupDataExtractorFactory.create(client, datafeed, job, response.getJobs(), factoryHandler);
+                    } else {
+                        listener.onFailure(new IllegalArgumentException("Aggregations are required when using Rollup indices"));
+                    }
+                }
+            },
+            e -> {
+                if (e instanceof IndexNotFoundException) {
+                    listener.onFailure(new ResourceNotFoundException("datafeed [" + datafeed.getId()
+                        + "] cannot retrieve data because index " + ((IndexNotFoundException)e).getIndex() + " does not exist"));
+                } else {
+                    listener.onFailure(e);
+                }
+            }
+        );
+
+        GetRollupIndexCapsAction.Request request = new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0]));
+
+        ClientHelper.executeAsyncWithOrigin(
+                client,
+                ClientHelper.ML_ORIGIN,
+                GetRollupIndexCapsAction.INSTANCE,
+                request,
+                getRollupIndexCapsActionHandler);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java
new file mode 100644
index 0000000000000..a520839652546
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
+import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Abstract class for aggregated data extractors, e.g. {@link RollupDataExtractor}
+ *
+ * @param <T> The request builder type for getting data from ElasticSearch
+ */
+abstract class AbstractAggregationDataExtractor<T extends ActionRequestBuilder<SearchRequest, SearchResponse, ?
+    extends ActionRequestBuilder<SearchRequest, SearchResponse, ?>>>
+    implements DataExtractor {
+
+    private static final Logger LOGGER = LogManager.getLogger(AbstractAggregationDataExtractor.class);
+
+    /**
+     * The number of key-value pairs written in each batch to process.
+     * This has to be a number that is small enough to allow for responsive
+     * cancelling and big enough to not cause overhead by calling the
+     * post data action too often. The value of 1000 was determined via
+     * such testing.
+     */
+    private static int BATCH_KEY_VALUE_PAIRS = 1000;
+
+    protected final Client client;
+    protected final AggregationDataExtractorContext context;
+    private boolean hasNext;
+    private boolean isCancelled;
+    private AggregationToJsonProcessor aggregationToJsonProcessor;
+    private ByteArrayOutputStream outputStream;
+
+    AbstractAggregationDataExtractor(Client client, AggregationDataExtractorContext dataExtractorContext) {
+        this.client = Objects.requireNonNull(client);
+        context = Objects.requireNonNull(dataExtractorContext);
+        hasNext = true;
+        isCancelled = false;
+        outputStream = new ByteArrayOutputStream();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return hasNext;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        return isCancelled;
+    }
+
+    @Override
+    public void cancel() {
+        LOGGER.debug("[{}] Data extractor received cancel request", context.jobId);
+        isCancelled = true;
+        hasNext = false;
+    }
+
+    @Override
+    public Optional<InputStream> next() throws IOException {
+        if (!hasNext()) {
+            throw new NoSuchElementException();
+        }
+
+        if (aggregationToJsonProcessor == null) {
+            Aggregations aggs = search();
+            if (aggs == null) {
+                hasNext = false;
+                return Optional.empty();
+            }
+            initAggregationProcessor(aggs);
+        }
+
+        return Optional.ofNullable(processNextBatch());
+    }
+
+    private Aggregations search() throws IOException {
+        LOGGER.debug("[{}] Executing aggregated search", context.jobId);
+        SearchResponse searchResponse = executeSearchRequest(buildSearchRequest(buildBaseSearchSource()));
+        LOGGER.debug("[{}] Search response was obtained", context.jobId);
+        ExtractorUtils.checkSearchWasSuccessful(context.jobId, searchResponse);
+        return validateAggs(searchResponse.getAggregations());
+    }
+
+    private void initAggregationProcessor(Aggregations aggs) throws IOException {
+        aggregationToJsonProcessor = new AggregationToJsonProcessor(context.timeField, context.fields, context.includeDocCount,
+            context.start);
+        aggregationToJsonProcessor.process(aggs);
+    }
+
+    protected SearchResponse executeSearchRequest(T searchRequestBuilder) {
+        return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get);
+    }
+
+    private SearchSourceBuilder buildBaseSearchSource() {
+        // For derivative aggregations the first bucket will always be null
+        // so query one extra histogram bucket back and hope there is data
+        // in that bucket
+        long histogramSearchStartTime = Math.max(0, context.start - ExtractorUtils.getHistogramIntervalMillis(context.aggs));
+
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
+            .size(0)
+            .query(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, histogramSearchStartTime, context.end));
+
+        context.aggs.getAggregatorFactories().forEach(searchSourceBuilder::aggregation);
+        context.aggs.getPipelineAggregatorFactories().forEach(searchSourceBuilder::aggregation);
+        return searchSourceBuilder;
+    }
+
+    protected abstract T buildSearchRequest(SearchSourceBuilder searchRequestBuilder);
+
+    private Aggregations validateAggs(@Nullable Aggregations aggs) {
+        if (aggs == null) {
+            return null;
+        }
+        List<Aggregation> aggsAsList = aggs.asList();
+        if (aggsAsList.isEmpty()) {
+            return null;
+        }
+        if (aggsAsList.size() > 1) {
+            throw new IllegalArgumentException("Multiple top level aggregations not supported; found: "
+                + aggsAsList.stream().map(Aggregation::getName).collect(Collectors.toList()));
+        }
+
+        return aggs;
+    }
+
+    private InputStream processNextBatch() throws IOException {
+        outputStream.reset();
+
+        hasNext = aggregationToJsonProcessor.writeDocs(BATCH_KEY_VALUE_PAIRS, outputStream);
+        return new ByteArrayInputStream(outputStream.toByteArray());
+    }
+
+    protected long getHistogramInterval() {
+        return ExtractorUtils.getHistogramIntervalMillis(context.aggs);
+    }
+
+    public AggregationDataExtractorContext getContext() {
+        return context;
+    }
+
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java
index d83865b751f50..0ca92162c9920 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java
@@ -5,28 +5,10 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
-import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Collectors;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 /**
  * An implementation that extracts data from elasticsearch using search with aggregations on a client.
@@ -34,132 +16,18 @@
  * stored and they are then processed in batches. Cancellation is supported between batches.
  * Note that this class is NOT thread-safe.
  */
-class AggregationDataExtractor implements DataExtractor {
-
-    private static final Logger LOGGER = Loggers.getLogger(AggregationDataExtractor.class);
-
-    /**
-     * The number of key-value pairs written in each batch to process.
-     * This has to be a number that is small enough to allow for responsive
-     * cancelling and big enough to not cause overhead by calling the
-     * post data action too often. The value of 1000 was determined via
-     * such testing.
-     */
-    private static int BATCH_KEY_VALUE_PAIRS = 1000;
-
-    private final Client client;
-    private final AggregationDataExtractorContext context;
-    private boolean hasNext;
-    private boolean isCancelled;
-    private AggregationToJsonProcessor aggregationToJsonProcessor;
-    private ByteArrayOutputStream outputStream;
+class AggregationDataExtractor extends AbstractAggregationDataExtractor<SearchRequestBuilder> {
 
     AggregationDataExtractor(Client client, AggregationDataExtractorContext dataExtractorContext) {
-        this.client = Objects.requireNonNull(client);
-        context = Objects.requireNonNull(dataExtractorContext);
-        hasNext = true;
-        isCancelled = false;
-        outputStream = new ByteArrayOutputStream();
+        super(client, dataExtractorContext);
     }
 
     @Override
-    public boolean hasNext() {
-        return hasNext;
-    }
-
-    @Override
-    public boolean isCancelled() {
-        return isCancelled;
-    }
-
-    @Override
-    public void cancel() {
-        LOGGER.trace("[{}] Data extractor received cancel request", context.jobId);
-        isCancelled = true;
-        hasNext = false;
-    }
-
-    @Override
-    public Optional<InputStream> next() throws IOException {
-        if (!hasNext()) {
-            throw new NoSuchElementException();
-        }
-
-        if (aggregationToJsonProcessor == null) {
-            Aggregations aggs = search();
-            if (aggs == null) {
-                hasNext = false;
-                return Optional.empty();
-            }
-            initAggregationProcessor(aggs);
-        }
-
-        return Optional.ofNullable(processNextBatch());
-    }
-
-    private Aggregations search() throws IOException {
-        LOGGER.debug("[{}] Executing aggregated search", context.jobId);
-        SearchResponse searchResponse = executeSearchRequest(buildSearchRequest());
-        LOGGER.debug("[{}] Search response was obtained", context.jobId);
-        ExtractorUtils.checkSearchWasSuccessful(context.jobId, searchResponse);
-        return validateAggs(searchResponse.getAggregations());
-    }
-
-    private void initAggregationProcessor(Aggregations aggs) throws IOException {
-        aggregationToJsonProcessor = new AggregationToJsonProcessor(context.timeField, context.fields, context.includeDocCount,
-                context.start);
-        aggregationToJsonProcessor.process(aggs);
-    }
-
-    protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) {
-        return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get);
-    }
-
-    private SearchRequestBuilder buildSearchRequest() {
-        // For derivative aggregations the first bucket will always be null
-        // so query one extra histogram bucket back and hope there is data
-        // in that bucket
-        long histogramSearchStartTime = Math.max(0, context.start - getHistogramInterval());
-
-        SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client)
-                .setIndices(context.indices)
-                .setTypes(context.types)
-                .setSize(0)
-                .setQuery(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, histogramSearchStartTime, context.end));
-
-        context.aggs.getAggregatorFactories().forEach(searchRequestBuilder::addAggregation);
-        context.aggs.getPipelineAggregatorFactories().forEach(searchRequestBuilder::addAggregation);
-        return searchRequestBuilder;
-    }
-
-    private Aggregations validateAggs(@Nullable Aggregations aggs) {
-        if (aggs == null) {
-            return null;
-        }
-        List<Aggregation> aggsAsList = aggs.asList();
-        if (aggsAsList.isEmpty()) {
-            return null;
-        }
-        if (aggsAsList.size() > 1) {
-            throw new IllegalArgumentException("Multiple top level aggregations not supported; found: "
-                    + aggsAsList.stream().map(Aggregation::getName).collect(Collectors.toList()));
-        }
-
-        return aggs;
-    }
-
-    private InputStream processNextBatch() throws IOException {
-        outputStream.reset();
-
-        hasNext = aggregationToJsonProcessor.writeDocs(BATCH_KEY_VALUE_PAIRS, outputStream);
-        return new ByteArrayInputStream(outputStream.toByteArray());
-    }
-
-    private long getHistogramInterval() {
-        return ExtractorUtils.getHistogramIntervalMillis(context.aggs);
-    }
+    protected SearchRequestBuilder buildSearchRequest(SearchSourceBuilder searchSourceBuilder) {
 
-    AggregationDataExtractorContext getContext() {
-        return context;
+        return new SearchRequestBuilder(client, SearchAction.INSTANCE)
+            .setSource(searchSourceBuilder)
+            .setIndices(context.indices)
+            .setTypes(context.types);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
index e481986504d60..31a939b5cc6cd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -42,7 +42,7 @@
  */
 class AggregationToJsonProcessor {
 
-    private static final Logger LOGGER = Loggers.getLogger(AggregationToJsonProcessor.class);
+    private static final Logger LOGGER = LogManager.getLogger(AggregationToJsonProcessor.class);
 
     private final String timeField;
     private final Set<String> fields;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java
new file mode 100644
index 0000000000000..f5de574e99a96
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
+
+/**
+ * An implementation that extracts data from elasticsearch using search with aggregations against rollup indexes on a client.
+ * The first time {@link #next()} is called, the search is executed. The result aggregations are
+ * stored and they are then processed in batches. Cancellation is supported between batches.
+ * Note that this class is NOT thread-safe.
+ */
+class RollupDataExtractor extends AbstractAggregationDataExtractor<RollupSearchAction.RequestBuilder> {
+
+    RollupDataExtractor(Client client, AggregationDataExtractorContext dataExtractorContext) {
+        super(client, dataExtractorContext);
+    }
+
+    @Override
+    protected RollupSearchAction.RequestBuilder buildSearchRequest(SearchSourceBuilder searchSourceBuilder) {
+        SearchRequest searchRequest = new SearchRequest().indices(context.indices).source(searchSourceBuilder);
+
+        return new RollupSearchAction.RequestBuilder(client, searchRequest);
+    }
+
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java
new file mode 100644
index 0000000000000..c8a96d6c306af
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java
@@ -0,0 +1,218 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.utils.Intervals;
+import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps;
+import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps.RollupFieldCaps;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
+import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils.getHistogramAggregation;
+import static org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils.getHistogramIntervalMillis;
+import static org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils.validateAndGetCalendarInterval;
+
+public class RollupDataExtractorFactory implements DataExtractorFactory {
+
+    private final Client client;
+    private final DatafeedConfig datafeedConfig;
+    private final Job job;
+
+    private RollupDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job) {
+        this.client = Objects.requireNonNull(client);
+        this.datafeedConfig = Objects.requireNonNull(datafeedConfig);
+        this.job = Objects.requireNonNull(job);
+    }
+
+    @Override
+    public DataExtractor newExtractor(long start, long end) {
+        long histogramInterval = datafeedConfig.getHistogramIntervalMillis();
+        AggregationDataExtractorContext dataExtractorContext = new AggregationDataExtractorContext(
+            job.getId(),
+            job.getDataDescription().getTimeField(),
+            job.getAnalysisConfig().analysisFields(),
+            datafeedConfig.getIndices(),
+            datafeedConfig.getTypes(),
+            datafeedConfig.getQuery(),
+            datafeedConfig.getAggregations(),
+            Intervals.alignToCeil(start, histogramInterval),
+            Intervals.alignToFloor(end, histogramInterval),
+            job.getAnalysisConfig().getSummaryCountFieldName().equals(DatafeedConfig.DOC_COUNT),
+            datafeedConfig.getHeaders());
+        return new RollupDataExtractor(client, dataExtractorContext);
+    }
+
+    public static void create(Client client,
+                              DatafeedConfig datafeed,
+                              Job job,
+                              Map<String, RollableIndexCaps> rollupJobsWithCaps,
+                              ActionListener<DataExtractorFactory> listener) {
+
+        final AggregationBuilder datafeedHistogramAggregation = getHistogramAggregation(
+            datafeed.getAggregations().getAggregatorFactories());
+        if ((datafeedHistogramAggregation instanceof DateHistogramAggregationBuilder) == false) {
+            listener.onFailure(
+                new IllegalArgumentException("Rollup requires that the datafeed configuration use a [date_histogram] aggregation," +
+                    " not a [histogram] aggregation over the time field."));
+            return;
+        }
+
+        final String timeField = ((ValuesSourceAggregationBuilder) datafeedHistogramAggregation).field();
+
+        Set<ParsedRollupCaps> rollupCapsSet = rollupJobsWithCaps.values()
+            .stream()
+            .flatMap(rollableIndexCaps -> rollableIndexCaps.getJobCaps().stream())
+            .map(rollupJobCaps -> ParsedRollupCaps.fromJobFieldCaps(rollupJobCaps.getFieldCaps(), timeField))
+            .collect(Collectors.toSet());
+
+        final long datafeedInterval = getHistogramIntervalMillis(datafeedHistogramAggregation);
+
+        List<ParsedRollupCaps> validIntervalCaps = rollupCapsSet.stream()
+            .filter(rollupCaps -> validInterval(datafeedInterval, rollupCaps))
+            .collect(Collectors.toList());
+
+        if (validIntervalCaps.isEmpty()) {
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "Rollup capabilities do not have a [date_histogram] aggregation with an interval " +
+                        "that is a multiple of the datafeed's interval.")
+            );
+            return;
+        }
+        final List<ValuesSourceAggregationBuilder> flattenedAggs = new ArrayList<>();
+        flattenAggregations(datafeed.getAggregations().getAggregatorFactories(), datafeedHistogramAggregation, flattenedAggs);
+
+        if (validIntervalCaps.stream().noneMatch(rollupJobConfig -> hasAggregations(rollupJobConfig, flattenedAggs))) {
+            listener.onFailure(
+                new IllegalArgumentException("Rollup capabilities do not support all the datafeed aggregations at the desired interval.")
+            );
+            return;
+        }
+
+        listener.onResponse(new RollupDataExtractorFactory(client, datafeed, job));
+    }
+
+    private static boolean validInterval(long datafeedInterval, ParsedRollupCaps rollupJobGroupConfig) {
+        if (rollupJobGroupConfig.hasDatehistogram() == false) {
+            return false;
+        }
+        if ("UTC".equalsIgnoreCase(rollupJobGroupConfig.getTimezone()) == false) {
+            return false;
+        }
+        try {
+            long jobInterval = validateAndGetCalendarInterval(rollupJobGroupConfig.getInterval());
+            return datafeedInterval % jobInterval == 0;
+        } catch (ElasticsearchStatusException exception) {
+            return false;
+        }
+    }
+
+    private static void flattenAggregations(final Collection<AggregationBuilder> datafeedAggregations,
+                                            final AggregationBuilder datafeedHistogramAggregation,
+                                            final List<ValuesSourceAggregationBuilder> flattenedAggregations) {
+        for (AggregationBuilder aggregationBuilder : datafeedAggregations) {
+            if (aggregationBuilder.equals(datafeedHistogramAggregation) == false) {
+                flattenedAggregations.add((ValuesSourceAggregationBuilder)aggregationBuilder);
+            }
+            flattenAggregations(aggregationBuilder.getSubAggregations(), datafeedHistogramAggregation, flattenedAggregations);
+        }
+    }
+
+    private static boolean hasAggregations(ParsedRollupCaps rollupCaps, List<ValuesSourceAggregationBuilder> datafeedAggregations) {
+        for (ValuesSourceAggregationBuilder aggregationBuilder : datafeedAggregations) {
+            String type = aggregationBuilder.getType();
+            String field = aggregationBuilder.field();
+            if (aggregationBuilder instanceof TermsAggregationBuilder) {
+                if (rollupCaps.supportedTerms.contains(field) == false) {
+                    return false;
+                }
+            } else {
+                if (rollupCaps.supportedMetrics.contains(field + "_" + type) == false) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    private static class ParsedRollupCaps {
+        private final Set<String> supportedMetrics;
+        private final Set<String> supportedTerms;
+        private final Map<String, Object> datehistogramAgg;
+        private static final List<String> aggsToIgnore =
+            Arrays.asList(HistogramAggregationBuilder.NAME, DateHistogramAggregationBuilder.NAME);
+
+        private static ParsedRollupCaps fromJobFieldCaps(Map<String, RollupFieldCaps> rollupFieldCaps, String timeField) {
+            Map<String, Object> datehistogram = null;
+            RollupFieldCaps timeFieldCaps = rollupFieldCaps.get(timeField);
+            if (timeFieldCaps != null) {
+                for(Map<String, Object> agg : timeFieldCaps.getAggs()) {
+                    if (agg.get("agg").equals(DateHistogramAggregationBuilder.NAME)) {
+                        datehistogram = agg;
+                    }
+                }
+            }
+            Set<String> supportedMetrics = new HashSet<>();
+            Set<String> supportedTerms = new HashSet<>();
+            rollupFieldCaps.forEach((field, fieldCaps) -> {
+                fieldCaps.getAggs().forEach(agg -> {
+                    String type = (String)agg.get("agg");
+                    if (type.equals(TermsAggregationBuilder.NAME)) {
+                        supportedTerms.add(field);
+                    } else if (aggsToIgnore.contains(type) == false) {
+                        supportedMetrics.add(field + "_" + type);
+                    }
+                });
+            });
+            return new ParsedRollupCaps(supportedMetrics, supportedTerms, datehistogram);
+        }
+
+        private ParsedRollupCaps(Set<String> supportedMetrics, Set<String> supportedTerms, Map<String, Object> datehistogramAgg) {
+            this.supportedMetrics = supportedMetrics;
+            this.supportedTerms = supportedTerms;
+            this.datehistogramAgg = datehistogramAgg;
+        }
+
+        private String getInterval() {
+            if (datehistogramAgg == null) {
+                return null;
+            }
+            return (String)datehistogramAgg.get(DateHistogramGroupConfig.INTERVAL);
+        }
+
+        private String getTimezone() {
+            if (datehistogramAgg == null) {
+                return null;
+            }
+            return (String)datehistogramAgg.get(DateHistogramGroupConfig.TIME_ZONE);
+        }
+
+        private boolean hasDatehistogram() {
+            return datehistogramAgg != null;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java
index 2e157c3d1e95a..89378699e2d90 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java
@@ -5,20 +5,26 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.chunked;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
 import org.elasticsearch.search.aggregations.metrics.min.Min;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
+import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
+import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.RollupDataExtractorFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -43,7 +49,14 @@
  */
 public class ChunkedDataExtractor implements DataExtractor {
 
-    private static final Logger LOGGER = Loggers.getLogger(ChunkedDataExtractor.class);
+    private interface DataSummary {
+        long estimateChunk();
+        boolean hasData();
+        long earliestTime();
+        long getDataTimeSpread();
+    }
+
+    private static final Logger LOGGER = LogManager.getLogger(ChunkedDataExtractor.class);
 
     private static final String EARLIEST_TIME = "earliest_time";
     private static final String LATEST_TIME = "latest_time";
@@ -54,6 +67,7 @@ public class ChunkedDataExtractor implements DataExtractor {
     private final Client client;
     private final DataExtractorFactory dataExtractorFactory;
     private final ChunkedDataExtractorContext context;
+    private final DataSummaryFactory dataSummaryFactory;
     private long currentStart;
     private long currentEnd;
     private long chunkSpan;
@@ -67,6 +81,7 @@ public ChunkedDataExtractor(Client client, DataExtractorFactory dataExtractorFac
         this.currentStart = context.start;
         this.currentEnd = context.start;
         this.isCancelled = false;
+        this.dataSummaryFactory = new DataSummaryFactory();
     }
 
     @Override
@@ -93,48 +108,24 @@ public Optional<InputStream> next() throws IOException {
     }
 
     private void setUpChunkedSearch() throws IOException {
-        DataSummary dataSummary = requestDataSummary();
-        if (dataSummary.totalHits > 0) {
-            currentStart = context.timeAligner.alignToFloor(dataSummary.earliestTime);
+        DataSummary dataSummary = dataSummaryFactory.buildDataSummary();
+        if (dataSummary.hasData()) {
+            currentStart = context.timeAligner.alignToFloor(dataSummary.earliestTime());
             currentEnd = currentStart;
             chunkSpan = context.chunkSpan == null ? dataSummary.estimateChunk() : context.chunkSpan.getMillis();
             chunkSpan = context.timeAligner.alignToCeil(chunkSpan);
-            LOGGER.debug("[{}]Chunked search configured:  totalHits = {}, dataTimeSpread = {} ms, chunk span = {} ms",
-                    context.jobId, dataSummary.totalHits, dataSummary.getDataTimeSpread(), chunkSpan);
+            LOGGER.debug("[{}]Chunked search configured: kind = {}, dataTimeSpread = {} ms, chunk span = {} ms",
+                    context.jobId, dataSummary.getClass().getSimpleName(), dataSummary.getDataTimeSpread(), chunkSpan);
         } else {
             // search is over
             currentEnd = context.end;
         }
     }
 
-    private DataSummary requestDataSummary() throws IOException {
-        SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client)
-                .setSize(0)
-                .setIndices(context.indices)
-                .setTypes(context.types)
-                .setQuery(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, currentStart, context.end))
-                .addAggregation(AggregationBuilders.min(EARLIEST_TIME).field(context.timeField))
-                .addAggregation(AggregationBuilders.max(LATEST_TIME).field(context.timeField));
-
-        SearchResponse response = executeSearchRequest(searchRequestBuilder);
-        LOGGER.debug("[{}] Data summary response was obtained", context.jobId);
-
-        ExtractorUtils.checkSearchWasSuccessful(context.jobId, response);
-
-        Aggregations aggregations = response.getAggregations();
-        long earliestTime = 0;
-        long latestTime = 0;
-        long totalHits = response.getHits().getTotalHits();
-        if (totalHits > 0) {
-            Min min = aggregations.get(EARLIEST_TIME);
-            earliestTime = (long) min.getValue();
-            Max max = aggregations.get(LATEST_TIME);
-            latestTime = (long) max.getValue();
-        }
-        return new DataSummary(earliestTime, latestTime, totalHits);
-    }
-
-    protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) {
+    protected SearchResponse executeSearchRequest(
+        ActionRequestBuilder<SearchRequest,
+            SearchResponse,
+            ? extends ActionRequestBuilder<SearchRequest, SearchResponse, ? >> searchRequestBuilder) {
         return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get);
     }
 
@@ -182,19 +173,104 @@ public void cancel() {
         isCancelled = true;
     }
 
-    private class DataSummary {
+    ChunkedDataExtractorContext getContext() {
+        return context;
+    }
+
+    private class DataSummaryFactory {
+
+        /**
+         * If there are aggregations, an AggregatedDataSummary object is created. It returns a ScrollingDataSummary otherwise.
+         *
+         * By default a DatafeedConfig with aggregations, should already have a manual ChunkingConfig created.
+         * However, the end user could have specifically set the ChunkingConfig to AUTO, which would not really work for aggregations.
+         * So, if we need to gather an appropriate chunked time for aggregations, we can utilize the AggregatedDataSummary
+         *
+         * @return DataSummary object
+         * @throws IOException when timefield range search fails
+         */
+        private DataSummary buildDataSummary() throws IOException {
+            return context.hasAggregations ? newAggregatedDataSummary() : newScrolledDataSummary();
+        }
+
+        private DataSummary newScrolledDataSummary() throws IOException {
+            SearchRequestBuilder searchRequestBuilder = rangeSearchRequest().setTypes(context.types);
+
+            SearchResponse response = executeSearchRequest(searchRequestBuilder);
+            LOGGER.debug("[{}] Scrolling Data summary response was obtained", context.jobId);
+
+            ExtractorUtils.checkSearchWasSuccessful(context.jobId, response);
+
+            Aggregations aggregations = response.getAggregations();
+            long earliestTime = 0;
+            long latestTime = 0;
+            long totalHits = response.getHits().getTotalHits();
+            if (totalHits > 0) {
+                Min min = aggregations.get(EARLIEST_TIME);
+                earliestTime = (long) min.getValue();
+                Max max = aggregations.get(LATEST_TIME);
+                latestTime = (long) max.getValue();
+            }
+            return new ScrolledDataSummary(earliestTime, latestTime, totalHits);
+        }
+
+        private DataSummary newAggregatedDataSummary() throws IOException {
+            // TODO: once RollupSearchAction is changed from indices:admin* to indices:data/read/* this branch is not needed
+            ActionRequestBuilder<SearchRequest,
+                SearchResponse,
+                ? extends ActionRequestBuilder<SearchRequest, SearchResponse, ?>>
+                searchRequestBuilder =
+                dataExtractorFactory instanceof RollupDataExtractorFactory ? rollupRangeSearchRequest() : rangeSearchRequest();
+            SearchResponse response = executeSearchRequest(searchRequestBuilder);
+            LOGGER.debug("[{}] Aggregating Data summary response was obtained", context.jobId);
+
+            ExtractorUtils.checkSearchWasSuccessful(context.jobId, response);
+
+            Aggregations aggregations = response.getAggregations();
+            Min min = aggregations.get(EARLIEST_TIME);
+            Max max = aggregations.get(LATEST_TIME);
+            return new AggregatedDataSummary(min.getValue(), max.getValue(), context.histogramInterval);
+        }
+
+        private SearchSourceBuilder rangeSearchBuilder() {
+            return new SearchSourceBuilder()
+                .size(0)
+                .query(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, currentStart, context.end))
+                .aggregation(AggregationBuilders.min(EARLIEST_TIME).field(context.timeField))
+                .aggregation(AggregationBuilders.max(LATEST_TIME).field(context.timeField));
+        }
+
+        private SearchRequestBuilder rangeSearchRequest() {
+            return new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                .setIndices(context.indices)
+                .setSource(rangeSearchBuilder());
+        }
+
+        private RollupSearchAction.RequestBuilder rollupRangeSearchRequest() {
+            SearchRequest searchRequest = new SearchRequest().indices(context.indices).source(rangeSearchBuilder());
+            return new RollupSearchAction.RequestBuilder(client, searchRequest);
+        }
+    }
+
+    private class ScrolledDataSummary implements DataSummary {
 
         private long earliestTime;
         private long latestTime;
         private long totalHits;
 
-        private DataSummary(long earliestTime, long latestTime, long totalHits) {
+        private ScrolledDataSummary(long earliestTime, long latestTime, long totalHits) {
             this.earliestTime = earliestTime;
             this.latestTime = latestTime;
             this.totalHits = totalHits;
         }
 
-        private long getDataTimeSpread() {
+        @Override
+        public long earliestTime() {
+            return earliestTime;
+        }
+
+        @Override
+        public long getDataTimeSpread() {
             return latestTime - earliestTime;
         }
 
@@ -206,7 +282,8 @@ private long getDataTimeSpread() {
          * However, assuming this as the chunk span may often lead to half-filled pages or empty searches.
          * It is beneficial to take a multiple of that. Based on benchmarking, we set this to 10x.
          */
-        private long estimateChunk() {
+        @Override
+        public long estimateChunk() {
             long dataTimeSpread = getDataTimeSpread();
             if (totalHits <= 0 || dataTimeSpread <= 0) {
                 return context.end - currentEnd;
@@ -214,9 +291,46 @@ private long estimateChunk() {
             long estimatedChunk = 10 * (context.scrollSize * getDataTimeSpread()) / totalHits;
             return Math.max(estimatedChunk, MIN_CHUNK_SPAN);
         }
+
+        @Override
+        public boolean hasData() {
+            return totalHits > 0;
+        }
     }
 
-    ChunkedDataExtractorContext getContext() {
-        return context;
+    private class AggregatedDataSummary implements DataSummary {
+
+        private final double earliestTime;
+        private final double latestTime;
+        private final long histogramIntervalMillis;
+
+        private AggregatedDataSummary(double earliestTime, double latestTime, long histogramInterval) {
+            this.earliestTime = earliestTime;
+            this.latestTime = latestTime;
+            this.histogramIntervalMillis = histogramInterval;
+        }
+
+        /**
+         * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder}
+         */
+        @Override
+        public long estimateChunk() {
+            return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis;
+        }
+
+        @Override
+        public boolean hasData() {
+            return (Double.isInfinite(earliestTime) || Double.isInfinite(latestTime)) == false;
+        }
+
+        @Override
+        public long earliestTime() {
+            return (long)earliestTime;
+        }
+
+        @Override
+        public long getDataTimeSpread() {
+            return (long)latestTime - (long)earliestTime;
+        }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java
index 38c2efd8679c0..bb32b40f7cde3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java
@@ -31,10 +31,13 @@ interface TimeAligner {
     final TimeValue chunkSpan;
     final TimeAligner timeAligner;
     final Map<String, String> headers;
+    final boolean hasAggregations;
+    final Long histogramInterval;
 
     ChunkedDataExtractorContext(String jobId, String timeField, List<String> indices, List<String> types,
                                 QueryBuilder query, int scrollSize, long start, long end, @Nullable TimeValue chunkSpan,
-                                TimeAligner timeAligner, Map<String, String> headers) {
+                                TimeAligner timeAligner, Map<String, String> headers, boolean hasAggregations,
+                                @Nullable Long histogramInterval) {
         this.jobId = Objects.requireNonNull(jobId);
         this.timeField = Objects.requireNonNull(timeField);
         this.indices = indices.toArray(new String[indices.size()]);
@@ -46,5 +49,7 @@ interface TimeAligner {
         this.chunkSpan = chunkSpan;
         this.timeAligner = Objects.requireNonNull(timeAligner);
         this.headers = headers;
+        this.hasAggregations = hasAggregations;
+        this.histogramInterval = histogramInterval;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java
index 7b5bac64740d6..67079cf2e6777 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java
@@ -42,7 +42,10 @@ public DataExtractor newExtractor(long start, long end) {
                 timeAligner.alignToFloor(end),
                 datafeedConfig.getChunkingConfig().getTimeSpan(),
                 timeAligner,
-                datafeedConfig.getHeaders());
+                datafeedConfig.getHeaders(),
+                datafeedConfig.hasAggregations(),
+                datafeedConfig.hasAggregations() ? datafeedConfig.getHistogramIntervalMillis() : null
+            );
         return new ChunkedDataExtractor(client, dataExtractorFactory, dataExtractorContext);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
similarity index 98%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
index f06eaaf813997..232cd53a359ce 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
 
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.search.SearchHit;
@@ -18,7 +18,7 @@
  * Represents a field to be extracted by the datafeed.
  * It encapsulates the extraction logic.
  */
-abstract class ExtractedField {
+public abstract class ExtractedField {
 
     public enum ExtractionMethod {
         SOURCE, DOC_VALUE, SCRIPT_FIELD
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
similarity index 51%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
index 2137141e4fba9..f9b2467fbcfd3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
@@ -3,18 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
 
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
 
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -25,20 +20,15 @@
 /**
  * The fields the datafeed has to extract
  */
-class ExtractedFields {
+public class ExtractedFields {
 
     private static final String TEXT = "text";
 
-    private final ExtractedField timeField;
     private final List<ExtractedField> allFields;
     private final List<ExtractedField> docValueFields;
     private final String[] sourceFields;
 
-    ExtractedFields(ExtractedField timeField, List<ExtractedField> allFields) {
-        if (!allFields.contains(timeField)) {
-            throw new IllegalArgumentException("timeField should also be contained in allFields");
-        }
-        this.timeField = Objects.requireNonNull(timeField);
+    public ExtractedFields(List<ExtractedField> allFields) {
         this.allFields = Collections.unmodifiableList(allFields);
         this.docValueFields = filterFields(ExtractedField.ExtractionMethod.DOC_VALUE, allFields);
         this.sourceFields = filterFields(ExtractedField.ExtractionMethod.SOURCE, allFields).stream().map(ExtractedField::getName)
@@ -61,60 +51,33 @@ private static List<ExtractedField> filterFields(ExtractedField.ExtractionMethod
         return fields.stream().filter(field -> field.getExtractionMethod() == method).collect(Collectors.toList());
     }
 
-    public String timeField() {
-        return timeField.getName();
+    public static ExtractedFields build(Collection<String> allFields, Set<String> scriptFields,
+                                        FieldCapabilitiesResponse fieldsCapabilities) {
+        ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(scriptFields, fieldsCapabilities);
+        return new ExtractedFields(allFields.stream().map(field -> extractionMethodDetector.detect(field)).collect(Collectors.toList()));
     }
 
-    public Long timeFieldValue(SearchHit hit) {
-        Object[] value = timeField.value(hit);
-        if (value.length != 1) {
-            throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a single value; actual was: "
-                    + Arrays.toString(value));
-        }
-        if (value[0] instanceof Long) {
-            return (Long) value[0];
-        }
-        throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a long value; actual was: " + value[0]);
-    }
+    protected static class ExtractionMethodDetector {
 
-    public static ExtractedFields build(Job job, DatafeedConfig datafeed, FieldCapabilitiesResponse fieldsCapabilities) {
-        Set<String> scriptFields = datafeed.getScriptFields().stream().map(sf -> sf.fieldName()).collect(Collectors.toSet());
-        ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(datafeed.getId(), scriptFields,
-                fieldsCapabilities);
-        String timeField = job.getDataDescription().getTimeField();
-        if (scriptFields.contains(timeField) == false && extractionMethodDetector.isAggregatable(timeField) == false) {
-            throw ExceptionsHelper.badRequestException("datafeed [" + datafeed.getId() + "] cannot retrieve time field [" + timeField
-                    + "] because it is not aggregatable");
-        }
-        ExtractedField timeExtractedField = ExtractedField.newTimeField(timeField, scriptFields.contains(timeField) ?
-                ExtractedField.ExtractionMethod.SCRIPT_FIELD : ExtractedField.ExtractionMethod.DOC_VALUE);
-        List<String> remainingFields = job.allInputFields().stream().filter(f -> !f.equals(timeField)).collect(Collectors.toList());
-        List<ExtractedField> allExtractedFields = new ArrayList<>(remainingFields.size() + 1);
-        allExtractedFields.add(timeExtractedField);
-        remainingFields.stream().forEach(field -> allExtractedFields.add(extractionMethodDetector.detect(field)));
-        return new ExtractedFields(timeExtractedField, allExtractedFields);
-    }
-
-    private static class ExtractionMethodDetector {
-
-        private final String datafeedId;
         private final Set<String> scriptFields;
         private final FieldCapabilitiesResponse fieldsCapabilities;
 
-        private ExtractionMethodDetector(String datafeedId, Set<String> scriptFields, FieldCapabilitiesResponse fieldsCapabilities) {
-            this.datafeedId = datafeedId;
+        protected ExtractionMethodDetector(Set<String> scriptFields, FieldCapabilitiesResponse fieldsCapabilities) {
             this.scriptFields = scriptFields;
             this.fieldsCapabilities = fieldsCapabilities;
         }
 
-        private ExtractedField detect(String field) {
+        protected ExtractedField detect(String field) {
             String internalField = field;
             ExtractedField.ExtractionMethod method = ExtractedField.ExtractionMethod.SOURCE;
             if (scriptFields.contains(field)) {
                 method = ExtractedField.ExtractionMethod.SCRIPT_FIELD;
             } else if (isAggregatable(field)) {
                 method = ExtractedField.ExtractionMethod.DOC_VALUE;
-            } else if (isText(field)) {
+                if (isFieldOfType(field, "date")) {
+                    return ExtractedField.newTimeField(field, method);
+                }
+            } else if (isFieldOfType(field, TEXT)) {
                 String parentField = MlStrings.getParentField(field);
                 // Field is text so check if it is a multi-field
                 if (Objects.equals(parentField, field) == false && fieldsCapabilities.getField(parentField) != null) {
@@ -127,11 +90,10 @@ private ExtractedField detect(String field) {
             return ExtractedField.newField(field, internalField, method);
         }
 
-        private boolean isAggregatable(String field) {
+        protected boolean isAggregatable(String field) {
             Map<String, FieldCapabilities> fieldCaps = fieldsCapabilities.getField(field);
             if (fieldCaps == null || fieldCaps.isEmpty()) {
-                throw ExceptionsHelper.badRequestException("datafeed [" + datafeedId + "] cannot retrieve field [" + field
-                        + "] because it has no mappings");
+                throw new IllegalArgumentException("cannot retrieve field [" + field + "] because it has no mappings");
             }
             for (FieldCapabilities capsPerIndex : fieldCaps.values()) {
                 if (!capsPerIndex.isAggregatable()) {
@@ -141,10 +103,10 @@ private boolean isAggregatable(String field) {
             return true;
         }
 
-        private boolean isText(String field) {
+        private boolean isFieldOfType(String field, String type) {
             Map<String, FieldCapabilities> fieldCaps = fieldsCapabilities.getField(field);
             if (fieldCaps != null && fieldCaps.size() == 1) {
-                return fieldCaps.containsKey(TEXT);
+                return fieldCaps.containsKey(type);
             }
             return false;
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFields.java
new file mode 100644
index 0000000000000..cf87671bf3301
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFields.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * The fields to extract for a datafeed that requires a time field
+ */
+public class TimeBasedExtractedFields extends ExtractedFields {
+
+    private final ExtractedField timeField;
+
+    public TimeBasedExtractedFields(ExtractedField timeField, List<ExtractedField> allFields) {
+        super(allFields);
+        if (!allFields.contains(timeField)) {
+            throw new IllegalArgumentException("timeField should also be contained in allFields");
+        }
+        this.timeField = Objects.requireNonNull(timeField);
+    }
+
+    public String timeField() {
+        return timeField.getName();
+    }
+
+    public Long timeFieldValue(SearchHit hit) {
+        Object[] value = timeField.value(hit);
+        if (value.length != 1) {
+            throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a single value; actual was: "
+                    + Arrays.toString(value));
+        }
+        if (value[0] instanceof Long) {
+            return (Long) value[0];
+        }
+        throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a long value; actual was: " + value[0]);
+    }
+
+    public static TimeBasedExtractedFields build(Job job, DatafeedConfig datafeed, FieldCapabilitiesResponse fieldsCapabilities) {
+        Set<String> scriptFields = datafeed.getScriptFields().stream().map(sf -> sf.fieldName()).collect(Collectors.toSet());
+        ExtractionMethodDetector extractionMethodDetector = new ExtractionMethodDetector(scriptFields, fieldsCapabilities);
+        String timeField = job.getDataDescription().getTimeField();
+        if (scriptFields.contains(timeField) == false && extractionMethodDetector.isAggregatable(timeField) == false) {
+            throw new IllegalArgumentException("cannot retrieve time field [" + timeField + "] because it is not aggregatable");
+        }
+        ExtractedField timeExtractedField = ExtractedField.newTimeField(timeField, scriptFields.contains(timeField) ?
+                ExtractedField.ExtractionMethod.SCRIPT_FIELD : ExtractedField.ExtractionMethod.DOC_VALUE);
+        List<String> remainingFields = job.allInputFields().stream().filter(f -> !f.equals(timeField)).collect(Collectors.toList());
+        List<ExtractedField> allExtractedFields = new ArrayList<>(remainingFields.size() + 1);
+        allExtractedFields.add(timeExtractedField);
+        remainingFields.stream().forEach(field -> allExtractedFields.add(extractionMethodDetector.detect(field)));
+        return new TimeBasedExtractedFields(timeExtractedField, allExtractedFields);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java
index c105597b3cdb1..49a5021f7401c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.ClearScrollRequest;
@@ -14,7 +15,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollAction;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -40,7 +41,7 @@
  */
 class ScrollDataExtractor implements DataExtractor {
 
-    private static final Logger LOGGER = Loggers.getLogger(ScrollDataExtractor.class);
+    private static final Logger LOGGER = LogManager.getLogger(ScrollDataExtractor.class);
     private static final TimeValue SCROLL_TIMEOUT = new TimeValue(30, TimeUnit.MINUTES);
 
     private final Client client;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java
index d1666497d241c..08e693849ec05 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.TimeBasedExtractedFields;
 
 import java.util.List;
 import java.util.Map;
@@ -15,7 +16,7 @@
 class ScrollDataExtractorContext {
 
     final String jobId;
-    final ExtractedFields extractedFields;
+    final TimeBasedExtractedFields extractedFields;
     final String[] indices;
     final String[] types;
     final QueryBuilder query;
@@ -25,7 +26,7 @@ class ScrollDataExtractorContext {
     final long end;
     final Map<String, String> headers;
 
-    ScrollDataExtractorContext(String jobId, ExtractedFields extractedFields, List<String> indices, List<String> types,
+    ScrollDataExtractorContext(String jobId, TimeBasedExtractedFields extractedFields, List<String> indices, List<String> types,
                                       QueryBuilder query, List<SearchSourceBuilder.ScriptField> scriptFields, int scrollSize,
                                       long start, long end, Map<String, String> headers) {
         this.jobId = Objects.requireNonNull(jobId);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
index 2c6e0deaebd9f..67689bd51b8b5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
@@ -16,8 +16,10 @@
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.TimeBasedExtractedFields;
 
 import java.util.Objects;
 
@@ -26,9 +28,9 @@ public class ScrollDataExtractorFactory implements DataExtractorFactory {
     private final Client client;
     private final DatafeedConfig datafeedConfig;
     private final Job job;
-    private final ExtractedFields extractedFields;
+    private final TimeBasedExtractedFields extractedFields;
 
-    private ScrollDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job, ExtractedFields extractedFields) {
+    private ScrollDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job, TimeBasedExtractedFields extractedFields) {
         this.client = Objects.requireNonNull(client);
         this.datafeedConfig = Objects.requireNonNull(datafeedConfig);
         this.job = Objects.requireNonNull(job);
@@ -56,12 +58,14 @@ public static void create(Client client, DatafeedConfig datafeed, Job job, Actio
         // Step 2. Contruct the factory and notify listener
         ActionListener<FieldCapabilitiesResponse> fieldCapabilitiesHandler = ActionListener.wrap(
                 fieldCapabilitiesResponse -> {
-                    ExtractedFields extractedFields = ExtractedFields.build(job, datafeed, fieldCapabilitiesResponse);
+                    TimeBasedExtractedFields extractedFields = TimeBasedExtractedFields.build(job, datafeed, fieldCapabilitiesResponse);
                     listener.onResponse(new ScrollDataExtractorFactory(client, datafeed, job, extractedFields));
                 }, e -> {
                     if (e instanceof IndexNotFoundException) {
                         listener.onFailure(new ResourceNotFoundException("datafeed [" + datafeed.getId()
                                 + "] cannot retrieve data because index " + ((IndexNotFoundException) e).getIndex() + " does not exist"));
+                    } else if (e instanceof IllegalArgumentException) {
+                        listener.onFailure(ExceptionsHelper.badRequestException("[" + datafeed.getId() + "] " + e.getMessage()));
                     } else {
                         listener.onFailure(e);
                     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java
index 52808ce397829..577d114d957e5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java
@@ -9,6 +9,8 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
 
 import java.io.IOException;
 import java.io.OutputStream;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
index 8a190048fd092..cc4131e7c65d9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.persistence;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
@@ -25,9 +27,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -72,7 +72,9 @@
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 
-public class DatafeedConfigProvider extends AbstractComponent {
+public class DatafeedConfigProvider {
+
+    private static final Logger logger = LogManager.getLogger(DatafeedConfigProvider.class);
 
     private final Client client;
     private final NamedXContentRegistry xContentRegistry;
@@ -83,8 +85,7 @@ public class DatafeedConfigProvider extends AbstractComponent {
         TO_XCONTENT_PARAMS.put(ToXContentParams.INCLUDE_TYPE, "true");
     }
 
-    public DatafeedConfigProvider(Client client, Settings settings, NamedXContentRegistry xContentRegistry) {
-        super(settings);
+    public DatafeedConfigProvider(Client client, NamedXContentRegistry xContentRegistry) {
         this.client = client;
         this.xContentRegistry = xContentRegistry;
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index 52516f59f1099..1097a547d2f57 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -84,6 +84,7 @@ public class JobManager extends AbstractComponent {
     private static final DeprecationLogger DEPRECATION_LOGGER =
             new DeprecationLogger(Loggers.getLogger(JobManager.class));
 
+    private final Settings settings;
     private final Environment environment;
     private final JobResultsProvider jobResultsProvider;
     private final ClusterService clusterService;
@@ -101,7 +102,7 @@ public class JobManager extends AbstractComponent {
     public JobManager(Environment environment, Settings settings, JobResultsProvider jobResultsProvider,
                       ClusterService clusterService, Auditor auditor, ThreadPool threadPool,
                       Client client, UpdateJobProcessNotifier updateJobProcessNotifier) {
-        super(settings);
+        this.settings = settings;
         this.environment = environment;
         this.jobResultsProvider = Objects.requireNonNull(jobResultsProvider);
         this.clusterService = Objects.requireNonNull(clusterService);
@@ -109,7 +110,7 @@ public JobManager(Environment environment, Settings settings, JobResultsProvider
         this.client = Objects.requireNonNull(client);
         this.threadPool = Objects.requireNonNull(threadPool);
         this.updateJobProcessNotifier = updateJobProcessNotifier;
-        this.jobConfigProvider = new JobConfigProvider(client, settings);
+        this.jobConfigProvider = new JobConfigProvider(client);
 
         maxModelMemoryLimit = MachineLearningField.MAX_MODEL_MEMORY_LIMIT.get(settings);
         clusterService.getClusterSettings()
@@ -551,7 +552,7 @@ public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionList
             ModelSnapshot modelSnapshot) {
 
         final ModelSizeStats modelSizeStats = modelSnapshot.getModelSizeStats();
-        final JobResultsPersister persister = new JobResultsPersister(settings, client);
+        final JobResultsPersister persister = new JobResultsPersister(client);
 
         // Step 3. After the model size stats is persisted, also persist the snapshot's quantiles and respond
         // -------
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/UpdateJobProcessNotifier.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/UpdateJobProcessNotifier.java
index c88e4f3d4408e..29e98d01ca9ac 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/UpdateJobProcessNotifier.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/UpdateJobProcessNotifier.java
@@ -5,16 +5,14 @@
  */
 package org.elasticsearch.xpack.ml.job;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
@@ -46,9 +44,9 @@
  * will fetch the valid state of those external resources ensuring the process is
  * in sync.
  */
-public class UpdateJobProcessNotifier extends AbstractComponent {
+public class UpdateJobProcessNotifier {
 
-    private static final Logger LOGGER = Loggers.getLogger(UpdateJobProcessNotifier.class);
+    private static final Logger logger = LogManager.getLogger(UpdateJobProcessNotifier.class);
 
     private final Client client;
     private final ClusterService clusterService;
@@ -57,8 +55,7 @@ public class UpdateJobProcessNotifier extends AbstractComponent {
 
     private volatile ThreadPool.Cancellable cancellable;
 
-    public UpdateJobProcessNotifier(Settings settings, Client client, ClusterService clusterService, ThreadPool threadPool) {
-        super(settings);
+    public UpdateJobProcessNotifier(Client client, ClusterService clusterService, ThreadPool threadPool) {
         this.client = client;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -112,7 +109,7 @@ void executeProcessUpdates(Iterator<UpdateHolder> updatesIterator) {
 
         if (update.isJobUpdate() && clusterService.localNode().isMasterNode() == false) {
             assert clusterService.localNode().isMasterNode();
-            LOGGER.error("Job update was submitted to non-master node [" + clusterService.getNodeName() + "]; update for job ["
+            logger.error("Job update was submitted to non-master node [" + clusterService.getNodeName() + "]; update for job ["
                     + update.getJobId() + "] will be ignored");
             executeProcessUpdates(updatesIterator);
             return;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java
index a0e00ebf73353..01fa529234bf4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/GrokPatternCreator.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.categorization;
 
-import org.elasticsearch.common.logging.Loggers;
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.grok.Grok;
 
 import java.util.ArrayList;
@@ -112,7 +112,7 @@ public static String findBestGrokMatchFromExamples(String jobId, String regex, C
                 // We should never get here.  If we do it implies a bug in the original categorization,
                 // as it's produced a regex that doesn't match the examples.
                 assert matcher.matches() : exampleProcessor.pattern() + " did not match " + example;
-                Loggers.getLogger(GrokPatternCreator.class).error("[{}] Pattern [{}] did not match example [{}]", jobId,
+                LogManager.getLogger(GrokPatternCreator.class).error("[{}] Pattern [{}] did not match example [{}]", jobId,
                         exampleProcessor.pattern(), example);
             }
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java
index e8655548592e1..371b7e05f6341 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -29,7 +29,7 @@
  * and iterate through them in batches.
  */
 public abstract class BatchedDocumentsIterator<T>  {
-    private static final Logger LOGGER = Loggers.getLogger(BatchedDocumentsIterator.class);
+    private static final Logger LOGGER = LogManager.getLogger(BatchedDocumentsIterator.class);
 
     private static final String CONTEXT_ALIVE_DURATION = "5m";
     private static final int BATCH_SIZE = 10000;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
index 64d825a09347b..e68c3a689c4bc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
@@ -32,9 +34,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -87,12 +87,13 @@
  * This class implements CRUD operation for the
  * anomaly detector job configuration document
  */
-public class JobConfigProvider extends AbstractComponent {
+public class JobConfigProvider {
+
+    private static final Logger logger = LogManager.getLogger(JobConfigProvider.class);
 
     private final Client client;
 
-    public JobConfigProvider(Client client, Settings settings) {
-        super(settings);
+    public JobConfigProvider(Client client) {
         this.client = client;
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
index d3b66f47661f2..c0b8b4d93659b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
@@ -34,8 +33,7 @@ public class JobDataCountsPersister extends AbstractComponent {
 
     private final Client client;
 
-    public JobDataCountsPersister(Settings settings, Client client) {
-        super(settings);
+    public JobDataCountsPersister(Client client) {
         this.client = client;
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
index 1e0825d14f90a..400ca28d97419 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkAction;
@@ -14,7 +15,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -36,7 +36,7 @@
 
 public class JobDataDeleter {
 
-    private static final Logger LOGGER = Loggers.getLogger(JobDataDeleter.class);
+    private static final Logger LOGGER = LogManager.getLogger(JobDataDeleter.class);
 
     private final Client client;
     private final String jobId;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
index f318737e0b474..5ee05feaebf7c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -49,8 +48,7 @@ public class JobRenormalizedResultsPersister extends AbstractComponent {
     private final Client client;
     private BulkRequest bulkRequest;
 
-    public JobRenormalizedResultsPersister(String jobId, Settings settings, Client client) {
-        super(settings);
+    public JobRenormalizedResultsPersister(String jobId, Client client) {
         this.jobId = jobId;
         this.client = client;
         bulkRequest = new BulkRequest();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
index 9efdbc1975716..85ac967a7284e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -68,8 +67,7 @@ public class JobResultsPersister extends AbstractComponent {
 
     private final Client client;
 
-    public JobResultsPersister(Settings settings, Client client) {
-        super(settings);
+    public JobResultsPersister(Client client) {
         this.client = client;
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index a2a75881e4f0c..7c42336a6f166 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
@@ -41,7 +42,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -129,7 +129,7 @@
 import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin;
 
 public class JobResultsProvider {
-    private static final Logger LOGGER = Loggers.getLogger(JobResultsProvider.class);
+    private static final Logger LOGGER = LogManager.getLogger(JobResultsProvider.class);
 
     private static final int RECORDS_SIZE_PARAM = 10000;
     public static final int BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE = 20;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamer.java
index a800ff95136e1..9a2c6a4938b2f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamer.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.ml.job.persistence;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
@@ -33,7 +33,7 @@
  */
 public class StateStreamer {
 
-    private static final Logger LOGGER = Loggers.getLogger(StateStreamer.class);
+    private static final Logger LOGGER = LogManager.getLogger(StateStreamer.class);
 
     private final Client client;
     private volatile boolean isCancelled;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
index 68a1acf78d54f..46d7f781d4e12 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
@@ -55,10 +54,7 @@ public class DataCountsReporter extends AbstractComponent {
 
     private DataStreamDiagnostics diagnostics;
 
-    public DataCountsReporter(Settings settings, Job job, DataCounts counts, JobDataCountsPersister dataCountsPersister) {
-
-        super(settings);
-
+    public DataCountsReporter(Job job, DataCounts counts, JobDataCountsPersister dataCountsPersister) {
         this.job = job;
         this.dataCountsPersister = dataCountsPersister;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
index d19617026034e..ac43c30cd3239 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -57,7 +57,7 @@
 
 public class AutodetectCommunicator implements Closeable {
 
-    private static final Logger LOGGER = Loggers.getLogger(AutodetectCommunicator.class);
+    private static final Logger LOGGER = LogManager.getLogger(AutodetectCommunicator.class);
     private static final Duration FLUSH_PROCESS_CHECK_FREQUENCY = Duration.ofSeconds(1);
 
     private final Job job;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
index f135c6f1051ae..24abeb9d45b47 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
@@ -108,6 +108,7 @@ public class AutodetectProcessManager extends AbstractComponent {
     public static final Setting<ByteSizeValue> MIN_DISK_SPACE_OFF_HEAP =
             Setting.byteSizeSetting("xpack.ml.min_disk_space_off_heap", new ByteSizeValue(5, ByteSizeUnit.GB), Property.NodeScope);
 
+    private final Settings settings;
     private final Client client;
     private final Environment environment;
     private final ThreadPool threadPool;
@@ -136,7 +137,7 @@ public AutodetectProcessManager(Environment environment, Settings settings, Clie
                                     JobDataCountsPersister jobDataCountsPersister,
                                     AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory,
                                     NamedXContentRegistry xContentRegistry, Auditor auditor) {
-        super(settings);
+        this.settings = settings;
         this.environment = environment;
         this.client = client;
         this.threadPool = threadPool;
@@ -510,12 +511,9 @@ AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodet
 
         // A TP with no queue, so that we fail immediately if there are no threads available
         ExecutorService autoDetectExecutorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME);
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings,
-            job,
-            autodetectParams.dataCounts(),
-            jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, autodetectParams.dataCounts(), jobDataCountsPersister);
         ScoresUpdater scoresUpdater = new ScoresUpdater(job, jobResultsProvider,
-                new JobRenormalizedResultsPersister(job.getId(), settings, client), normalizerFactory);
+                new JobRenormalizedResultsPersister(job.getId(), client), normalizerFactory);
         ExecutorService renormalizerExecutorService = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME);
         Renormalizer renormalizer = new ShortCircuitingRenormalizer(jobId, scoresUpdater,
                 renormalizerExecutorService);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
index ea31c5de4dffa..3185ebc6f1c7d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.internal.io.IOUtils;
@@ -33,7 +33,7 @@
 
 public class NativeAutodetectProcessFactory implements AutodetectProcessFactory {
 
-    private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcessFactory.class);
+    private static final Logger LOGGER = LogManager.getLogger(NativeAutodetectProcessFactory.class);
     private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper();
     public static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(10);
 
@@ -68,7 +68,7 @@ public AutodetectProcess createAutodetectProcess(Job job,
         int numberOfFields = job.allInputFields().size() + (includeTokensField ? 1 : 0) + 1;
 
         AutodetectStateProcessor stateProcessor = new AutodetectStateProcessor(client, job.getId());
-        AutodetectResultsParser resultsParser = new AutodetectResultsParser(settings);
+        AutodetectResultsParser resultsParser = new AutodetectResultsParser();
         NativeAutodetectProcess autodetect = new NativeAutodetectProcess(
                 job.getId(), processPipes.getLogStream().get(), processPipes.getProcessInStream().get(),
                 processPipes.getProcessOutStream().get(), processPipes.getRestoreStream().orElse(null), numberOfFields,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/ProcessContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/ProcessContext.java
index 474968adc932f..8771712dfde13 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/ProcessContext.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/ProcessContext.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.ml.action.TransportOpenJobAction.JobTask;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -21,7 +21,7 @@
  */
 final class ProcessContext {
 
-    private static final Logger LOGGER = Loggers.getLogger(ProcessContext.class);
+    private static final Logger LOGGER = LogManager.getLogger(ProcessContext.class);
 
     private final ReentrantLock lock = new ReentrantLock();
     private final JobTask jobTask;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
index a3614a2353e90..9ae247cca73b6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect.output;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
@@ -17,7 +18,6 @@
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -82,7 +82,7 @@
  */
 public class AutoDetectResultProcessor {
 
-    private static final Logger LOGGER = Loggers.getLogger(AutoDetectResultProcessor.class);
+    private static final Logger LOGGER = LogManager.getLogger(AutoDetectResultProcessor.class);
 
     /**
      * This is how far behind real-time we'll update the job with the latest established model memory.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java
index d4f9c431f7bdf..8d7e64c3d45d2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -27,11 +26,6 @@
  * start array symbol and the data must be terminated with the end array symbol.
  */
 public class AutodetectResultsParser extends AbstractComponent {
-
-    public AutodetectResultsParser(Settings settings) {
-        super(settings);
-    }
-
     public Iterator<AutodetectResult> parseResults(InputStream in) throws ElasticsearchParseException {
         try {
             XContentParser parser = XContentFactory.xContent(XContentType.JSON)
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java
index 8734a88336876..887b43d0b5927 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@@ -41,7 +41,7 @@
  */
 class CsvDataToProcessWriter extends AbstractDataToProcessWriter {
 
-    private static final Logger LOGGER = Loggers.getLogger(CsvDataToProcessWriter.class);
+    private static final Logger LOGGER = LogManager.getLogger(CsvDataToProcessWriter.class);
 
     /**
      * Maximum number of lines allowed within a single CSV record.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
index 15a162c6a3c00..92fe2c3b0b50a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -37,7 +37,7 @@
  */
 class JsonDataToProcessWriter extends AbstractDataToProcessWriter {
 
-    private static final Logger LOGGER = Loggers.getLogger(JsonDataToProcessWriter.class);
+    private static final Logger LOGGER = LogManager.getLogger(JsonDataToProcessWriter.class);
     private NamedXContentRegistry xContentRegistry;
 
     JsonDataToProcessWriter(boolean includeControlField, boolean includeTokensField, AutodetectProcess autodetectProcess,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/diagnostics/DataStreamDiagnostics.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/diagnostics/DataStreamDiagnostics.java
index a225587d0bb75..f491f90c9bb08 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/diagnostics/DataStreamDiagnostics.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/diagnostics/DataStreamDiagnostics.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.ml.job.process.diagnostics;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 
@@ -23,7 +23,7 @@ public class DataStreamDiagnostics {
      */
     private static final int DATA_SPARSITY_THRESHOLD = 2;
 
-    private static final Logger LOGGER = Loggers.getLogger(DataStreamDiagnostics.class);
+    private static final Logger LOGGER = LogManager.getLogger(DataStreamDiagnostics.class);
 
     private final BucketDiagnostics bucketDiagnostics;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
index 5d320a1bd715c..652d575b07ab0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java
@@ -27,7 +27,6 @@
  * - It can be used to produce results in testing that do not vary based on changes to the real normalization algorithms
  */
 public class MultiplyingNormalizerProcess implements NormalizerProcess {
-
     private static final Logger LOGGER = LogManager.getLogger(MultiplyingNormalizerProcess.class);
 
     private final double factor;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
index 74eb01987c562..755e3da102f63 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.ml.job.process.normalizer;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler;
 
 import java.io.IOException;
@@ -29,7 +29,7 @@
  * and in exactly the same order as the inputs.
  */
 public class Normalizer {
-    private static final Logger LOGGER = Loggers.getLogger(Normalizer.class);
+    private static final Logger LOGGER = LogManager.getLogger(Normalizer.class);
 
     private final String jobId;
     private final NormalizerProcessFactory processFactory;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
index c2ef2fab7f8a4..47ab2364db67e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java
@@ -5,16 +5,16 @@
  */
 package org.elasticsearch.xpack.ml.job.process.normalizer;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.ml.job.persistence.BatchedDocumentsIterator;
-import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister;
+import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.util.ArrayList;
 import java.util.Deque;
@@ -27,7 +27,7 @@
  * with the renormalized scores
  */
 public class ScoresUpdater {
-    private static final Logger LOGGER = Loggers.getLogger(ScoresUpdater.class);
+    private static final Logger LOGGER = LogManager.getLogger(ScoresUpdater.class);
 
     /**
      * Target number of buckets to renormalize at a time
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
index 7db66387db8e0..0bd5a11609d6d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.ml.job.process.normalizer;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
 
 import java.util.Deque;
@@ -21,7 +21,7 @@
  */
 public class ShortCircuitingRenormalizer implements Renormalizer {
 
-    private static final Logger LOGGER = Loggers.getLogger(ShortCircuitingRenormalizer.class);
+    private static final Logger LOGGER = LogManager.getLogger(ShortCircuitingRenormalizer.class);
 
     private final String jobId;
     private final ScoresUpdater scoresUpdater;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
index bb14a5fa1bdee..981d257afa1a0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.retention;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
@@ -13,7 +14,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -55,7 +55,7 @@
  */
 public class ExpiredForecastsRemover implements MlDataRemover {
 
-    private static final Logger LOGGER = Loggers.getLogger(ExpiredForecastsRemover.class);
+    private static final Logger LOGGER = LogManager.getLogger(ExpiredForecastsRemover.class);
     private static final int MAX_FORECASTS = 10000;
     private static final String RESULTS_INDEX_PATTERN =  AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*";
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
index 2272121a68ff3..74ff0c8dc8776 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.retention;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
@@ -14,7 +15,6 @@
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -45,7 +45,7 @@
  */
 public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover {
 
-    private static final Logger LOGGER = Loggers.getLogger(ExpiredModelSnapshotsRemover.class);
+    private static final Logger LOGGER = LogManager.getLogger(ExpiredModelSnapshotsRemover.class);
 
     /**
      *  The max number of snapshots to fetch per job. It is set to 10K, the default for an index as
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
index dfa9d66814ebb..d7b16ef8c468e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.ml.job.retention;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -42,7 +42,7 @@
  */
 public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover {
 
-    private static final Logger LOGGER = Loggers.getLogger(ExpiredResultsRemover.class);
+    private static final Logger LOGGER = LogManager.getLogger(ExpiredResultsRemover.class);
 
     private final Client client;
     private final Auditor auditor;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
index fd4085d202041..ea094dfe6b4f1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/UnusedStateRemover.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.job.retention;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -12,7 +13,6 @@
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
@@ -35,7 +35,7 @@
  */
 public class UnusedStateRemover implements MlDataRemover {
 
-    private static final Logger LOGGER = Loggers.getLogger(UnusedStateRemover.class);
+    private static final Logger LOGGER = LogManager.getLogger(UnusedStateRemover.class);
 
     private final Client client;
     private final ClusterService clusterService;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java
index 458ad2a8fa3e1..addf478708a9e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.ml.notifications;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -27,7 +27,7 @@
 
 public class Auditor {
 
-    private static final Logger LOGGER = Loggers.getLogger(Auditor.class);
+    private static final Logger LOGGER = LogManager.getLogger(Auditor.class);
 
     private final Client client;
     private final String nodeName;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
index 747074028953c..721e07721e32d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ml.process;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
@@ -27,8 +27,9 @@
 /**
  * Maintains the connection to the native controller daemon that can start other processes.
  */
+@SuppressWarnings("ALL")
 public class NativeController {
-    private static final Logger LOGGER = Loggers.getLogger(NativeController.class);
+    private static final Logger LOGGER = LogManager.getLogger(NativeController.class);
 
     /**
      * Process controller native program name
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java
index 9670fadfefff3..1f9aee7bbeea3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java
@@ -6,8 +6,8 @@
 
 package org.elasticsearch.xpack.ml.process;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
@@ -21,7 +21,7 @@
  */
 public class NativeStorageProvider {
 
-    private static final Logger LOGGER = Loggers.getLogger(NativeStorageProvider.class);
+    private static final Logger LOGGER = LogManager.getLogger(NativeStorageProvider.class);
 
 
     private static final String LOCAL_STORAGE_SUBFOLDER = "ml-local-data";
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
index 341b9ae371b82..01638d7c68fc0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandler.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.process.logging;
 
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
@@ -13,7 +14,6 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -47,7 +47,7 @@
  */
 public class CppLogMessageHandler implements Closeable {
 
-    private static final Logger LOGGER = Loggers.getLogger(CppLogMessageHandler.class);
+    private static final Logger LOGGER = LogManager.getLogger(CppLogMessageHandler.class);
     private static final int DEFAULT_READBUF_SIZE = 1024;
     private static final int DEFAULT_ERROR_STORE_SIZE = 5;
     private static final long MAX_MESSAGE_INTERVAL_SECONDS = 10;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
index 485556d8441ea..8cf2c9e47b1bb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/LocalStateMachineLearning.java
@@ -5,15 +5,31 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.monitoring.Monitoring;
 import org.elasticsearch.xpack.security.Security;
 
 import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
 
 public class LocalStateMachineLearning extends LocalStateCompositeXPackPlugin {
 
@@ -50,6 +66,45 @@ protected XPackLicenseState getLicenseState() {
             @Override
             protected XPackLicenseState getLicenseState() { return thisVar.getLicenseState(); }
         });
+        plugins.add(new MockedRollupPlugin());
     }
-}
 
+    /**
+     * This is only required as we now have to have the GetRollupIndexCapsAction as a valid action in our node.
+     * The MachineLearningLicenseTests attempt to create a datafeed referencing this LocalStateMachineLearning object.
+     * Consequently, we need to be able to take this rollup action (response does not matter)
+     * as the datafeed extractor now depends on it.
+     */
+    public static class MockedRollupPlugin extends Plugin implements ActionPlugin {
+
+        @Override
+        public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+            return Collections.singletonList(
+                new ActionHandler<>(GetRollupIndexCapsAction.INSTANCE, MockedRollupIndexCapsTransport.class)
+            );
+        }
+
+        public static class MockedRollupIndexCapsTransport
+            extends TransportAction<GetRollupIndexCapsAction.Request, GetRollupIndexCapsAction.Response> {
+
+            @Inject
+            public MockedRollupIndexCapsTransport(Settings settings, TransportService transportService,
+                                                  ThreadPool threadPool, IndexNameExpressionResolver indexNameExpressionResolver) {
+                super(settings, GetRollupIndexCapsAction.NAME, threadPool, new ActionFilters(new HashSet<>()),
+                    indexNameExpressionResolver, transportService.getTaskManager());
+            }
+
+            @Override
+            protected void doExecute(Task task,
+                                     GetRollupIndexCapsAction.Request request,
+                                     ActionListener<GetRollupIndexCapsAction.Response> listener) {
+                listener.onResponse(new GetRollupIndexCapsAction.Response());
+            }
+
+            @Override
+            protected void doExecute(GetRollupIndexCapsAction.Request request, ActionListener<GetRollupIndexCapsAction.Response> listener) {
+                listener.onResponse(new GetRollupIndexCapsAction.Response());
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java
index 2defead100774..90a1d45f9e15e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlAssignmentNotifierTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
@@ -35,7 +34,7 @@ public class MlAssignmentNotifierTests extends ESTestCase {
     public void testClusterChanged_info() {
         Auditor auditor = mock(Auditor.class);
         ClusterService clusterService = mock(ClusterService.class);
-        MlAssignmentNotifier notifier = new MlAssignmentNotifier(Settings.EMPTY, auditor, clusterService);
+        MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, clusterService);
         notifier.onMaster();
 
         DiscoveryNode node =
@@ -63,7 +62,7 @@ public void testClusterChanged_info() {
     public void testClusterChanged_warning() {
         Auditor auditor = mock(Auditor.class);
         ClusterService clusterService = mock(ClusterService.class);
-        MlAssignmentNotifier notifier = new MlAssignmentNotifier(Settings.EMPTY, auditor, clusterService);
+        MlAssignmentNotifier notifier = new MlAssignmentNotifier(auditor, clusterService);
         notifier.onMaster();
 
         ClusterState previous = ClusterState.builder(new ClusterName("_name"))
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
index c7f50440f0e54..5ded1b205a110 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -62,7 +61,7 @@ public void setUpMocks() {
     }
 
     public void testInitialize() {
-        MlInitializationService initializationService = new MlInitializationService(Settings.EMPTY, threadPool, clusterService, client);
+        MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client);
 
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
                 .nodes(DiscoveryNodes.builder()
@@ -77,7 +76,7 @@ public void testInitialize() {
     }
 
     public void testInitialize_noMasterNode() {
-        MlInitializationService initializationService = new MlInitializationService(Settings.EMPTY, threadPool, clusterService, client);
+        MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client);
 
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
                 .nodes(DiscoveryNodes.builder()
@@ -90,7 +89,7 @@ public void testInitialize_noMasterNode() {
     }
 
     public void testInitialize_alreadyInitialized() {
-        MlInitializationService initializationService = new MlInitializationService(Settings.EMPTY, threadPool, clusterService, client);
+        MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client);
 
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
                 .nodes(DiscoveryNodes.builder()
@@ -108,7 +107,7 @@ public void testInitialize_alreadyInitialized() {
     }
 
     public void testNodeGoesFromMasterToNonMasterAndBack() {
-        MlInitializationService initializationService = new MlInitializationService(Settings.EMPTY, threadPool, clusterService, client);
+        MlInitializationService initializationService = new MlInitializationService(threadPool, clusterService, client);
         MlDailyMaintenanceService initialDailyMaintenanceService = mock(MlDailyMaintenanceService.class);
         initializationService.setDailyMaintenanceService(initialDailyMaintenanceService);
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
index 52e38a70abdb5..7399d31f7c6ac 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
@@ -14,23 +14,39 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
+import org.elasticsearch.xpack.core.rollup.action.RollableIndexCaps;
+import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps;
+import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.GroupConfig;
+import org.elasticsearch.xpack.core.rollup.job.MetricConfig;
+import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
+import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 import org.elasticsearch.xpack.ml.datafeed.DatafeedManagerTests;
 import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationDataExtractorFactory;
 import org.elasticsearch.xpack.ml.datafeed.extractor.chunked.ChunkedDataExtractorFactory;
+import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.RollupDataExtractorFactory;
 import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorFactory;
-import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.junit.Before;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.same;
@@ -41,6 +57,7 @@
 public class DataExtractorFactoryTests extends ESTestCase {
 
     private FieldCapabilitiesResponse fieldsCapabilities;
+    private GetRollupIndexCapsAction.Response getRollupIndexResponse;
 
     private Client client;
 
@@ -54,12 +71,22 @@ public void setUpTests() {
         givenAggregatableField("time", "date");
         givenAggregatableField("field", "keyword");
 
+        getRollupIndexResponse = mock(GetRollupIndexCapsAction.Response.class);
+        when(getRollupIndexResponse.getJobs()).thenReturn(new HashMap<>());
+
         doAnswer(invocationMock -> {
             @SuppressWarnings("raw_types")
             ActionListener listener = (ActionListener) invocationMock.getArguments()[2];
             listener.onResponse(fieldsCapabilities);
             return null;
         }).when(client).execute(same(FieldCapabilitiesAction.INSTANCE), any(), any());
+
+        doAnswer(invocationMock -> {
+            @SuppressWarnings("raw_types")
+            ActionListener listener = (ActionListener) invocationMock.getArguments()[2];
+            listener.onResponse(getRollupIndexResponse);
+            return null;
+        }).when(client).execute(same(GetRollupIndexCapsAction.INSTANCE), any(), any());
     }
 
     public void testCreateDataExtractorFactoryGivenDefaultScroll() {
@@ -165,6 +192,162 @@ public void testCreateDataExtractorFactoryGivenDefaultAggregationWithAutoChunk()
         DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
     }
 
+    public void testCreateDataExtractorFactoryGivenRollupAndValidAggregation() {
+        givenAggregatableRollup("myField", "max", 5, "termField");
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newOff());
+        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+        MaxAggregationBuilder myField = AggregationBuilders.max("myField").field("myField");
+        TermsAggregationBuilder myTerm = AggregationBuilders.terms("termAgg").field("termField").subAggregation(myField);
+        datafeedConfig.setAggregations(AggregatorFactories.builder().addAggregator(
+            AggregationBuilders.dateHistogram("time").interval(600_000).subAggregation(maxTime).subAggregation(myTerm).field("time")));
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> assertThat(dataExtractorFactory, instanceOf(RollupDataExtractorFactory.class)),
+            e -> fail()
+        );
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    public void testCreateDataExtractorFactoryGivenRollupAndValidAggregationAndAutoChunk() {
+        givenAggregatableRollup("myField", "max", 5, "termField");
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newAuto());
+        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+        MaxAggregationBuilder myField = AggregationBuilders.max("myField").field("myField");
+        TermsAggregationBuilder myTerm = AggregationBuilders.terms("termAgg").field("termField").subAggregation(myField);
+        datafeedConfig.setAggregations(AggregatorFactories.builder().addAggregator(
+            AggregationBuilders.dateHistogram("time").interval(600_000).subAggregation(maxTime).subAggregation(myTerm).field("time")));
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> assertThat(dataExtractorFactory, instanceOf(ChunkedDataExtractorFactory.class)),
+            e -> fail()
+        );
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    public void testCreateDataExtractorFactoryGivenRollupButNoAggregations() {
+        givenAggregatableRollup("myField", "max", 5);
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newOff());
+
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> fail(),
+            e -> {
+                assertThat(e.getMessage(), equalTo("Aggregations are required when using Rollup indices"));
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+            }
+        );
+
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    public void testCreateDataExtractorFactoryGivenRollupWithBadInterval() {
+        givenAggregatableRollup("myField", "max", 7, "termField");
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newOff());
+        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+        MaxAggregationBuilder myField = AggregationBuilders.max("myField").field("myField");
+        TermsAggregationBuilder myTerm = AggregationBuilders.terms("termAgg").field("termField").subAggregation(myField);
+        datafeedConfig.setAggregations(AggregatorFactories.builder().addAggregator(
+            AggregationBuilders.dateHistogram("time").interval(600_000).subAggregation(maxTime).subAggregation(myTerm).field("time")));
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> fail(),
+            e -> {
+                assertThat(e.getMessage(),
+                    containsString("Rollup capabilities do not have a [date_histogram] aggregation with an interval " +
+                        "that is a multiple of the datafeed's interval."));
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+            }
+        );
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    public void testCreateDataExtractorFactoryGivenRollupMissingTerms() {
+        givenAggregatableRollup("myField", "max", 5);
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newOff());
+        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+        MaxAggregationBuilder myField = AggregationBuilders.max("myField").field("myField");
+        TermsAggregationBuilder myTerm = AggregationBuilders.terms("termAgg").field("termField").subAggregation(myField);
+        datafeedConfig.setAggregations(AggregatorFactories.builder().addAggregator(
+            AggregationBuilders.dateHistogram("time").interval(600_000).subAggregation(maxTime).subAggregation(myTerm).field("time")));
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> fail(),
+            e -> {
+                assertThat(e.getMessage(),
+                    containsString("Rollup capabilities do not support all the datafeed aggregations at the desired interval."));
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+            }
+        );
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    public void testCreateDataExtractorFactoryGivenRollupMissingMetric() {
+        givenAggregatableRollup("myField", "max", 5, "termField");
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setTimeField("time");
+        Job.Builder jobBuilder = DatafeedManagerTests.createDatafeedJob();
+        jobBuilder.setDataDescription(dataDescription);
+        DatafeedConfig.Builder datafeedConfig = DatafeedManagerTests.createDatafeedConfig("datafeed1", "foo");
+        datafeedConfig.setChunkingConfig(ChunkingConfig.newOff());
+        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
+        MaxAggregationBuilder myField = AggregationBuilders.max("myField").field("otherField");
+        TermsAggregationBuilder myTerm = AggregationBuilders.terms("termAgg").field("termField").subAggregation(myField);
+        datafeedConfig.setAggregations(AggregatorFactories.builder().addAggregator(
+            AggregationBuilders.dateHistogram("time").interval(600_000).subAggregation(maxTime).subAggregation(myTerm).field("time")));
+        ActionListener<DataExtractorFactory> listener = ActionListener.wrap(
+            dataExtractorFactory -> fail(),
+            e -> {
+                assertThat(e.getMessage(),
+                    containsString("Rollup capabilities do not support all the datafeed aggregations at the desired interval."));
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+            }
+        );
+        DataExtractorFactory.create(client, datafeedConfig.build(), jobBuilder.build(new Date()), listener);
+    }
+
+    private void givenAggregatableRollup(String field, String type, int minuteInterval, String... groupByTerms) {
+        List<MetricConfig> metricConfigs = Arrays.asList(new MetricConfig(field, Collections.singletonList(type)),
+            new MetricConfig("time", Arrays.asList("min", "max")));
+        TermsGroupConfig termsGroupConfig = null;
+        if (groupByTerms.length > 0) {
+            termsGroupConfig = new TermsGroupConfig(groupByTerms);
+        }
+        RollupJobConfig rollupJobConfig = new RollupJobConfig("rollupJob1",
+            "myIndexes*",
+            "myIndex_rollup",
+            "*/30 * * * * ?",
+            300,
+            new GroupConfig(
+                new DateHistogramGroupConfig("time", DateHistogramInterval.minutes(minuteInterval)), null, termsGroupConfig),
+            metricConfigs,
+            null);
+        RollupJobCaps rollupJobCaps = new RollupJobCaps(rollupJobConfig);
+        RollableIndexCaps rollableIndexCaps = new RollableIndexCaps("myIndex_rollup", Collections.singletonList(rollupJobCaps));
+        Map<String, RollableIndexCaps> jobs = new HashMap<>(1);
+        jobs.put("rollupJob1", rollableIndexCaps);
+        when(getRollupIndexResponse.getJobs()).thenReturn(jobs);
+    }
+
     private void givenAggregatableField(String field, String type) {
         FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
         when(fieldCaps.isSearchable()).thenReturn(true);
@@ -173,4 +356,4 @@ private void givenAggregatableField(String field, String type) {
         fieldCapsMap.put(type, fieldCaps);
         when(fieldsCapabilities.getField(field)).thenReturn(fieldCapsMap);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java
index 903ab4af1157f..af2b3e7750b63 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java
@@ -46,7 +46,7 @@
 
 public class AggregationDataExtractorTests extends ESTestCase {
 
-    private Client client;
+    private Client testClient;
     private List<SearchRequestBuilder> capturedSearchRequests;
     private String jobId;
     private String timeField;
@@ -61,7 +61,7 @@ private class TestDataExtractor extends AggregationDataExtractor {
         private SearchResponse nextResponse;
 
         TestDataExtractor(long start, long end) {
-            super(client, createContext(start, end));
+            super(testClient, createContext(start, end));
         }
 
         @Override
@@ -77,7 +77,7 @@ void setNextResponse(SearchResponse searchResponse) {
 
     @Before
     public void setUpTests() {
-        client = mock(Client.class);
+        testClient = mock(Client.class);
         capturedSearchRequests = new ArrayList<>();
         jobId = "test-job";
         timeField = "time";
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java
index 18c35155b6f5d..766f35fce6ff0 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.chunked;
 
-import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.Client;
@@ -44,7 +45,7 @@
 public class ChunkedDataExtractorTests extends ESTestCase {
 
     private Client client;
-    private List<SearchRequestBuilder> capturedSearchRequests;
+    private List<SearchRequest> capturedSearchRequests;
     private String jobId;
     private String timeField;
     private List<String> types;
@@ -62,9 +63,15 @@ private class TestDataExtractor extends ChunkedDataExtractor {
             super(client, dataExtractorFactory, createContext(start, end));
         }
 
+        TestDataExtractor(long start, long end, boolean hasAggregations, Long histogramInterval) {
+            super(client, dataExtractorFactory, createContext(start, end, hasAggregations, histogramInterval));
+        }
+
         @Override
-        protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) {
-            capturedSearchRequests.add(searchRequestBuilder);
+        protected SearchResponse executeSearchRequest(
+            ActionRequestBuilder<SearchRequest, SearchResponse, ? extends ActionRequestBuilder<SearchRequest,SearchResponse,?>>
+                searchRequestBuilder) {
+            capturedSearchRequests.add(searchRequestBuilder.request());
             return nextResponse;
         }
 
@@ -136,6 +143,89 @@ public void testExtractionGivenSpecifiedChunk() throws IOException {
         assertThat(searchRequest, not(containsString("\"sort\"")));
     }
 
+    public void testExtractionGivenSpecifiedChunkAndAggs() throws IOException {
+        chunkSpan = TimeValue.timeValueSeconds(1);
+        TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L, true, 1000L);
+        extractor.setNextResponse(createSearchResponse(0L, 1000L, 2200L));
+
+        InputStream inputStream1 = mock(InputStream.class);
+        InputStream inputStream2 = mock(InputStream.class);
+        InputStream inputStream3 = mock(InputStream.class);
+
+        DataExtractor subExtactor1 = new StubSubExtractor(inputStream1, inputStream2);
+        when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1);
+
+        DataExtractor subExtactor2 = new StubSubExtractor(inputStream3);
+        when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtactor2);
+
+        assertThat(extractor.hasNext(), is(true));
+        assertEquals(inputStream1, extractor.next().get());
+        assertThat(extractor.hasNext(), is(true));
+        assertEquals(inputStream2, extractor.next().get());
+        assertThat(extractor.hasNext(), is(true));
+        assertEquals(inputStream3, extractor.next().get());
+        assertThat(extractor.hasNext(), is(true));
+        assertThat(extractor.next().isPresent(), is(false));
+
+        verify(dataExtractorFactory).newExtractor(1000L, 2000L);
+        verify(dataExtractorFactory).newExtractor(2000L, 2300L);
+        Mockito.verifyNoMoreInteractions(dataExtractorFactory);
+
+        assertThat(capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("\"size\":0"));
+        assertThat(searchRequest, containsString("\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," +
+            "{\"range\":{\"time\":{\"from\":1000,\"to\":2300,\"include_lower\":true,\"include_upper\":false," +
+            "\"format\":\"epoch_millis\",\"boost\":1.0}}}]"));
+        assertThat(searchRequest, containsString("\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," +
+            "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}"));
+        assertThat(searchRequest, not(containsString("\"sort\"")));
+    }
+
+    public void testExtractionGivenAutoChunkAndAggs() throws IOException {
+        chunkSpan = null;
+        TestDataExtractor extractor = new TestDataExtractor(100_000L, 450_000L, true, 200L);
+
+        extractor.setNextResponse(createSearchResponse(0L, 100_000L, 400_000L));
+
+        InputStream inputStream1 = mock(InputStream.class);
+        InputStream inputStream2 = mock(InputStream.class);
+
+        // 200 * 1_000 == 200_000
+        DataExtractor subExtactor1 = new StubSubExtractor(inputStream1);
+        when(dataExtractorFactory.newExtractor(100_000L, 300_000L)).thenReturn(subExtactor1);
+
+        DataExtractor subExtactor2 = new StubSubExtractor(inputStream2);
+        when(dataExtractorFactory.newExtractor(300_000L, 450_000L)).thenReturn(subExtactor2);
+
+        assertThat(extractor.hasNext(), is(true));
+        assertEquals(inputStream1, extractor.next().get());
+        assertThat(extractor.hasNext(), is(true));
+        assertEquals(inputStream2, extractor.next().get());
+        assertThat(extractor.next().isPresent(), is(false));
+        assertThat(extractor.hasNext(), is(false));
+
+        verify(dataExtractorFactory).newExtractor(100_000L, 300_000L);
+        verify(dataExtractorFactory).newExtractor(300_000L, 450_000L);
+        Mockito.verifyNoMoreInteractions(dataExtractorFactory);
+
+        assertThat(capturedSearchRequests.size(), equalTo(1));
+    }
+
+    public void testExtractionGivenAutoChunkAndAggsAndNoData() throws IOException {
+        chunkSpan = null;
+        TestDataExtractor extractor = new TestDataExtractor(100L, 500L, true, 200L);
+
+        extractor.setNextResponse(createNullSearchResponse());
+
+        assertThat(extractor.next().isPresent(), is(false));
+        assertThat(extractor.hasNext(), is(false));
+
+        Mockito.verifyNoMoreInteractions(dataExtractorFactory);
+
+        assertThat(capturedSearchRequests.size(), equalTo(1));
+    }
+
     public void testExtractionGivenAutoChunkAndScrollSize1000() throws IOException {
         chunkSpan = null;
         scrollSize = 1000;
@@ -430,6 +520,27 @@ private SearchResponse createSearchResponse(long totalHits, long earliestTime, l
         return searchResponse;
     }
 
+    private SearchResponse createNullSearchResponse() {
+        SearchResponse searchResponse = mock(SearchResponse.class);
+        when(searchResponse.status()).thenReturn(RestStatus.OK);
+        SearchHit[] hits = new SearchHit[0];
+        SearchHits searchHits = new SearchHits(hits, 0, 1);
+        when(searchResponse.getHits()).thenReturn(searchHits);
+
+        List<Aggregation> aggs = new ArrayList<>();
+        Min min = mock(Min.class);
+        when(min.getValue()).thenReturn(Double.POSITIVE_INFINITY);
+        when(min.getName()).thenReturn("earliest_time");
+        aggs.add(min);
+        Max max = mock(Max.class);
+        when(max.getValue()).thenReturn(Double.POSITIVE_INFINITY);
+        when(max.getName()).thenReturn("latest_time");
+        aggs.add(max);
+        Aggregations aggregations = new Aggregations(aggs) {};
+        when(searchResponse.getAggregations()).thenReturn(aggregations);
+        return searchResponse;
+    }
+
     private SearchResponse createErrorResponse() {
         SearchResponse searchResponse = mock(SearchResponse.class);
         when(searchResponse.status()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR);
@@ -445,8 +556,12 @@ private SearchResponse createResponseWithShardFailures() {
     }
 
     private ChunkedDataExtractorContext createContext(long start, long end) {
+        return createContext(start, end, false, null);
+    }
+
+    private ChunkedDataExtractorContext createContext(long start, long end, boolean hasAggregations, Long histogramInterval) {
         return new ChunkedDataExtractorContext(jobId, timeField, indices, types, query, scrollSize, start, end, chunkSpan,
-                ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap());
+                ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap(), hasAggregations, histogramInterval);
     }
 
     private static class StubSubExtractor implements DataExtractor {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
similarity index 98%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
index d29769b607ee6..1e5e6fa652db1 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
@@ -3,11 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
 
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
 import org.elasticsearch.xpack.ml.test.SearchHitBuilder;
 import org.joda.time.DateTime;
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java
new file mode 100644
index 0000000000000..222531141364a
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ExtractedFieldsTests extends ESTestCase {
+
+    public void testAllTypesOfFields() {
+        ExtractedField docValue1 = ExtractedField.newField("doc1", ExtractedField.ExtractionMethod.DOC_VALUE);
+        ExtractedField docValue2 = ExtractedField.newField("doc2", ExtractedField.ExtractionMethod.DOC_VALUE);
+        ExtractedField scriptField1 = ExtractedField.newField("scripted1", ExtractedField.ExtractionMethod.SCRIPT_FIELD);
+        ExtractedField scriptField2 = ExtractedField.newField("scripted2", ExtractedField.ExtractionMethod.SCRIPT_FIELD);
+        ExtractedField sourceField1 = ExtractedField.newField("src1", ExtractedField.ExtractionMethod.SOURCE);
+        ExtractedField sourceField2 = ExtractedField.newField("src2", ExtractedField.ExtractionMethod.SOURCE);
+        ExtractedFields extractedFields = new ExtractedFields(Arrays.asList(
+                docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2));
+
+        assertThat(extractedFields.getAllFields().size(), equalTo(6));
+        assertThat(extractedFields.getDocValueFields().stream().map(ExtractedField::getName).toArray(String[]::new),
+            equalTo(new String[] {"doc1", "doc2"}));
+        assertThat(extractedFields.getSourceFields(), equalTo(new String[] {"src1", "src2"}));
+    }
+
+    public void testBuildGivenMixtureOfTypes() {
+        Map<String, FieldCapabilities> timeCaps = new HashMap<>();
+        timeCaps.put("date", createFieldCaps(true));
+        Map<String, FieldCapabilities> valueCaps = new HashMap<>();
+        valueCaps.put("float", createFieldCaps(true));
+        valueCaps.put("keyword", createFieldCaps(true));
+        Map<String, FieldCapabilities> airlineCaps = new HashMap<>();
+        airlineCaps.put("text", createFieldCaps(false));
+        FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
+        when(fieldCapabilitiesResponse.getField("time")).thenReturn(timeCaps);
+        when(fieldCapabilitiesResponse.getField("value")).thenReturn(valueCaps);
+        when(fieldCapabilitiesResponse.getField("airline")).thenReturn(airlineCaps);
+
+        ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("time", "value", "airline", "airport"),
+            new HashSet<>(Collections.singletonList("airport")), fieldCapabilitiesResponse);
+
+        assertThat(extractedFields.getDocValueFields().size(), equalTo(2));
+        assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time"));
+        assertThat(extractedFields.getDocValueFields().get(0).getDocValueFormat(), equalTo("epoch_millis"));
+        assertThat(extractedFields.getDocValueFields().get(1).getName(), equalTo("value"));
+        assertThat(extractedFields.getDocValueFields().get(1).getDocValueFormat(), equalTo(DocValueFieldsContext.USE_DEFAULT_FORMAT));
+        assertThat(extractedFields.getSourceFields(), equalTo(new String[] {"airline"}));
+        assertThat(extractedFields.getAllFields().size(), equalTo(4));
+    }
+
+    public void testBuildGivenMultiFields() {
+        Job.Builder jobBuilder = new Job.Builder("foo");
+        jobBuilder.setDataDescription(new DataDescription.Builder());
+        Detector.Builder detector = new Detector.Builder("count", null);
+        detector.setByFieldName("airline.text");
+        detector.setOverFieldName("airport.keyword");
+        jobBuilder.setAnalysisConfig(new AnalysisConfig.Builder(Collections.singletonList(detector.build())));
+
+        DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("feed", jobBuilder.getId());
+        datafeedBuilder.setIndices(Collections.singletonList("foo"));
+
+        Map<String, FieldCapabilities> text = new HashMap<>();
+        text.put("text", createFieldCaps(false));
+        Map<String, FieldCapabilities> keyword = new HashMap<>();
+        keyword.put("keyword", createFieldCaps(true));
+        FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
+        when(fieldCapabilitiesResponse.getField("airline")).thenReturn(text);
+        when(fieldCapabilitiesResponse.getField("airline.text")).thenReturn(text);
+        when(fieldCapabilitiesResponse.getField("airport")).thenReturn(text);
+        when(fieldCapabilitiesResponse.getField("airport.keyword")).thenReturn(keyword);
+
+        ExtractedFields extractedFields = ExtractedFields.build(Arrays.asList("airline.text", "airport.keyword"),
+                Collections.emptySet(), fieldCapabilitiesResponse);
+
+        assertThat(extractedFields.getDocValueFields().size(), equalTo(1));
+        assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("airport.keyword"));
+        assertThat(extractedFields.getSourceFields().length, equalTo(1));
+        assertThat(extractedFields.getSourceFields()[0], equalTo("airline"));
+        assertThat(extractedFields.getAllFields().size(), equalTo(2));
+
+        assertThat(extractedFields.getAllFields().stream().filter(f -> f.getName().equals("airport.keyword")).findFirst().get().getAlias(),
+                equalTo("airport.keyword"));
+        assertThat(extractedFields.getAllFields().stream().filter(f -> f.getName().equals("airline")).findFirst().get().getAlias(),
+                equalTo("airline.text"));
+    }
+
+    public void testBuildGivenFieldWithoutMappings() {
+        FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ExtractedFields.build(
+                Collections.singletonList("value"), Collections.emptySet(), fieldCapabilitiesResponse));
+        assertThat(e.getMessage(), equalTo("cannot retrieve field [value] because it has no mappings"));
+    }
+
+    private static FieldCapabilities createFieldCaps(boolean isAggregatable) {
+        FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
+        when(fieldCaps.isAggregatable()).thenReturn(isAggregatable);
+        return fieldCaps;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java
similarity index 80%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java
index d029cfea590f1..5e388afad282a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java
@@ -3,12 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.datafeed.extractor.scroll;
+package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
 
-import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
-import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
@@ -31,16 +29,16 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class ExtractedFieldsTests extends ESTestCase {
+public class TimeBasedExtractedFieldsTests extends ESTestCase {
 
     private ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE);
 
     public void testInvalidConstruction() {
-        expectThrows(IllegalArgumentException.class, () -> new ExtractedFields(timeField, Collections.emptyList()));
+        expectThrows(IllegalArgumentException.class, () -> new TimeBasedExtractedFields(timeField, Collections.emptyList()));
     }
 
     public void testTimeFieldOnly() {
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Arrays.asList(timeField));
 
         assertThat(extractedFields.getAllFields(), equalTo(Arrays.asList(timeField)));
         assertThat(extractedFields.timeField(), equalTo("time"));
@@ -56,7 +54,7 @@ public void testAllTypesOfFields() {
         ExtractedField scriptField2 = ExtractedField.newField("scripted2", ExtractedField.ExtractionMethod.SCRIPT_FIELD);
         ExtractedField sourceField1 = ExtractedField.newField("src1", ExtractedField.ExtractionMethod.SOURCE);
         ExtractedField sourceField2 = ExtractedField.newField("src2", ExtractedField.ExtractionMethod.SOURCE);
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField,
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Arrays.asList(timeField,
                 docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2));
 
         assertThat(extractedFields.getAllFields().size(), equalTo(7));
@@ -67,31 +65,31 @@ public void testAllTypesOfFields() {
     }
 
     public void testTimeFieldValue() {
-        final long millis = randomLong();
-        final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", new DateTime(millis)).build();
-        final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField));
+        long millis = randomLong();
+        SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", new DateTime(millis)).build();
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Collections.singletonList(timeField));
         assertThat(extractedFields.timeFieldValue(hit), equalTo(millis));
     }
 
     public void testStringTimeFieldValue() {
-        final long millis = randomLong();
-        final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", Long.toString(millis)).build();
-        final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField));
+        long millis = randomLong();
+        SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", Long.toString(millis)).build();
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Collections.singletonList(timeField));
         assertThat(extractedFields.timeFieldValue(hit), equalTo(millis));
     }
 
     public void testPre6xTimeFieldValue() {
         // Prior to 6.x, timestamps were simply `long` milliseconds-past-the-epoch values
-        final long millis = randomLong();
-        final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", millis).build();
-        final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField));
+        long millis = randomLong();
+        SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", millis).build();
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Collections.singletonList(timeField));
         assertThat(extractedFields.timeFieldValue(hit), equalTo(millis));
     }
 
     public void testTimeFieldValueGivenEmptyArray() {
         SearchHit hit = new SearchHitBuilder(1).addField("time", Collections.emptyList()).build();
 
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Arrays.asList(timeField));
 
         expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit));
     }
@@ -99,7 +97,7 @@ public void testTimeFieldValueGivenEmptyArray() {
     public void testTimeFieldValueGivenValueHasTwoElements() {
         SearchHit hit = new SearchHitBuilder(1).addField("time", Arrays.asList(1L, 2L)).build();
 
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Arrays.asList(timeField));
 
         expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit));
     }
@@ -107,7 +105,7 @@ public void testTimeFieldValueGivenValueHasTwoElements() {
     public void testTimeFieldValueGivenValueIsString() {
         SearchHit hit = new SearchHitBuilder(1).addField("time", "a string").build();
 
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField, Arrays.asList(timeField));
 
         expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit));
     }
@@ -137,7 +135,7 @@ public void testBuildGivenMixtureOfTypes() {
         when(fieldCapabilitiesResponse.getField("value")).thenReturn(valueCaps);
         when(fieldCapabilitiesResponse.getField("airline")).thenReturn(airlineCaps);
 
-        ExtractedFields extractedFields = ExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(),
+        TimeBasedExtractedFields extractedFields = TimeBasedExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(),
                 fieldCapabilitiesResponse);
 
         assertThat(extractedFields.timeField(), equalTo("time"));
@@ -175,7 +173,7 @@ public void testBuildGivenMultiFields() {
         when(fieldCapabilitiesResponse.getField("airport")).thenReturn(text);
         when(fieldCapabilitiesResponse.getField("airport.keyword")).thenReturn(keyword);
 
-        ExtractedFields extractedFields = ExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(),
+        TimeBasedExtractedFields extractedFields = TimeBasedExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(),
                 fieldCapabilitiesResponse);
 
         assertThat(extractedFields.timeField(), equalTo("time"));
@@ -209,10 +207,9 @@ public void testBuildGivenTimeFieldIsNotAggregatable() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
         when(fieldCapabilitiesResponse.getField("time")).thenReturn(timeCaps);
 
-        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-                () -> ExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
-        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        assertThat(e.getMessage(), equalTo("datafeed [feed] cannot retrieve time field [time] because it is not aggregatable"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> TimeBasedExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
+        assertThat(e.getMessage(), equalTo("cannot retrieve time field [time] because it is not aggregatable"));
     }
 
     public void testBuildGivenTimeFieldIsNotAggregatableInSomeIndices() {
@@ -231,10 +228,9 @@ public void testBuildGivenTimeFieldIsNotAggregatableInSomeIndices() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
         when(fieldCapabilitiesResponse.getField("time")).thenReturn(timeCaps);
 
-        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-                () -> ExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
-        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        assertThat(e.getMessage(), equalTo("datafeed [feed] cannot retrieve time field [time] because it is not aggregatable"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> TimeBasedExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
+        assertThat(e.getMessage(), equalTo("cannot retrieve time field [time] because it is not aggregatable"));
     }
 
     public void testBuildGivenFieldWithoutMappings() {
@@ -252,10 +248,9 @@ public void testBuildGivenFieldWithoutMappings() {
         FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class);
         when(fieldCapabilitiesResponse.getField("time")).thenReturn(timeCaps);
 
-        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-                () -> ExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
-        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
-        assertThat(e.getMessage(), equalTo("datafeed [feed] cannot retrieve field [value] because it has no mappings"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> TimeBasedExtractedFields.build(jobBuilder.build(new Date()), datafeedBuilder.build(), fieldCapabilitiesResponse));
+        assertThat(e.getMessage(), equalTo("cannot retrieve field [value] because it has no mappings"));
     }
 
     private static FieldCapabilities createFieldCaps(boolean isAggregatable) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java
index f72ae9b46b13f..93a76c5402b36 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java
@@ -27,6 +27,8 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.TimeBasedExtractedFields;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
@@ -61,7 +63,7 @@ public class ScrollDataExtractorTests extends ESTestCase {
     private List<String> capturedContinueScrollIds;
     private ArgumentCaptor<ClearScrollRequest> capturedClearScrollRequests;
     private String jobId;
-    private ExtractedFields extractedFields;
+    private TimeBasedExtractedFields extractedFields;
     private List<String> types;
     private List<String> indices;
     private QueryBuilder query;
@@ -128,7 +130,7 @@ public void setUpTests() {
         capturedContinueScrollIds = new ArrayList<>();
         jobId = "test-job";
         ExtractedField timeField = ExtractedField.newField("time", ExtractedField.ExtractionMethod.DOC_VALUE);
-        extractedFields = new ExtractedFields(timeField,
+        extractedFields = new TimeBasedExtractedFields(timeField,
                 Arrays.asList(timeField, ExtractedField.newField("field_1", ExtractedField.ExtractionMethod.DOC_VALUE)));
         indices = Arrays.asList("index-1", "index-2");
         types = Arrays.asList("type-1", "type-2");
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java
index 60e14023d3685..41a74814461c9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java
@@ -7,6 +7,9 @@
 
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.TimeBasedExtractedFields;
 import org.elasticsearch.xpack.ml.test.SearchHitBuilder;
 
 import java.io.ByteArrayOutputStream;
@@ -23,7 +26,8 @@ public void testProcessGivenSingleHit() throws IOException {
         ExtractedField missingField = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.DOC_VALUE);
         ExtractedField singleField = ExtractedField.newField("single", ExtractedField.ExtractionMethod.DOC_VALUE);
         ExtractedField arrayField = ExtractedField.newField("array", ExtractedField.ExtractionMethod.DOC_VALUE);
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField, missingField, singleField, arrayField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField,
+                Arrays.asList(timeField, missingField, singleField, arrayField));
 
         SearchHit hit = new SearchHitBuilder(8)
                 .addField("time", 1000L)
@@ -41,7 +45,8 @@ public void testProcessGivenMultipleHits() throws IOException {
         ExtractedField missingField = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.DOC_VALUE);
         ExtractedField singleField = ExtractedField.newField("single", ExtractedField.ExtractionMethod.DOC_VALUE);
         ExtractedField arrayField = ExtractedField.newField("array", ExtractedField.ExtractionMethod.DOC_VALUE);
-        ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField, missingField, singleField, arrayField));
+        TimeBasedExtractedFields extractedFields = new TimeBasedExtractedFields(timeField,
+                Arrays.asList(timeField, missingField, singleField, arrayField));
 
         SearchHit hit1 = new SearchHitBuilder(8)
                 .addField("time", 1000L)
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureTestCase.java
index 0b9d491fb7077..f47a3582c66a9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureTestCase.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.filestructurefinder;
 
-import org.elasticsearch.common.logging.Loggers;
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -80,7 +80,7 @@ public void initExplanation() {
 
     @After
     public void printExplanation() {
-        Loggers.getLogger(getClass()).info("Explanation:\n" + String.join("\n", explanation));
+        LogManager.getLogger(getClass()).info("Explanation:\n" + String.join("\n", explanation));
     }
 
     protected Boolean randomHasByteOrderMarker(String charset) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index 072f3941660fe..18668cb8bef2d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -89,7 +89,7 @@ public void createComponents() throws Exception {
         renormalizer = mock(Renormalizer.class);
         capturedUpdateModelSnapshotOnJobRequests = new ArrayList<>();
         resultProcessor = new AutoDetectResultProcessor(client(), auditor, JOB_ID, renormalizer,
-                new JobResultsPersister(nodeSettings(), client()), jobResultsProvider, new ModelSizeStats.Builder(JOB_ID).build(), false) {
+                new JobResultsPersister(client()), jobResultsProvider, new ModelSizeStats.Builder(JOB_ID).build(), false) {
             @Override
             protected void updateModelSnapshotIdOnJob(ModelSnapshot modelSnapshot) {
                 capturedUpdateModelSnapshotOnJobRequests.add(modelSnapshot);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java
index 09d8c7ed41888..9496f4ca0d8f2 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedConfigProviderIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -47,7 +46,7 @@ public class DatafeedConfigProviderIT extends MlSingleNodeTestCase {
 
     @Before
     public void createComponents() throws Exception {
-        datafeedConfigProvider = new DatafeedConfigProvider(client(), Settings.EMPTY, xContentRegistry());
+        datafeedConfigProvider = new DatafeedConfigProvider(client(), xContentRegistry());
         waitForMlTemplates();
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
index df3af13f71403..a026d5d6c337b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
@@ -33,7 +33,7 @@ public class EstablishedMemUsageIT extends BaseMlIntegTestCase {
     public void createComponents() {
         Settings settings = nodeSettings(0);
         jobResultsProvider = new JobResultsProvider(client(), settings);
-        jobResultsPersister = new JobResultsPersister(settings, client());
+        jobResultsPersister = new JobResultsPersister(client());
     }
 
     public void testEstablishedMem_givenNoResults() throws Exception {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java
index 6c5ceebc23af3..bd6e2df8ad0a6 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobConfigProviderIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -58,7 +57,7 @@ public class JobConfigProviderIT extends MlSingleNodeTestCase {
 
     @Before
     public void createComponents() throws Exception {
-        jobConfigProvider = new JobConfigProvider(client(), Settings.EMPTY);
+        jobConfigProvider = new JobConfigProvider(client());
         waitForMlTemplates();
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
index 1bf43a0dc72c0..3843181a0bc3c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
@@ -513,7 +513,7 @@ private void indexScheduledEvents(List<ScheduledEvent> events) throws IOExceptio
     }
 
     private void indexDataCounts(DataCounts counts, String jobId) throws Exception {
-        JobDataCountsPersister persister = new JobDataCountsPersister(nodeSettings(), client());
+        JobDataCountsPersister persister = new JobDataCountsPersister(client());
 
         AtomicReference<Exception> errorHolder = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
@@ -553,17 +553,17 @@ private void indexFilters(List<MlFilter> filters) throws IOException {
     }
 
     private void indexModelSizeStats(ModelSizeStats modelSizeStats) {
-        JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client());
+        JobResultsPersister persister = new JobResultsPersister(client());
         persister.persistModelSizeStats(modelSizeStats);
     }
 
     private void indexModelSnapshot(ModelSnapshot snapshot) {
-        JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client());
+        JobResultsPersister persister = new JobResultsPersister(client());
         persister.persistModelSnapshot(snapshot, WriteRequest.RefreshPolicy.IMMEDIATE);
     }
 
     private void indexQuantiles(Quantiles quantiles) {
-        JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client());
+        JobResultsPersister persister = new JobResultsPersister(client());
         persister.persistQuantiles(quantiles);
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java
index b1ec3008d33c9..f9bacee003c8b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ml.job.process.normalizer.BucketNormalizable;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
@@ -45,7 +44,7 @@ public void testExecuteRequestResetsBulkRequest() {
     public void testBulkRequestExecutesWhenReachMaxDocs() {
         BulkResponse bulkResponse = mock(BulkResponse.class);
         Client client = new MockClientBuilder("cluster").bulk(bulkResponse).build();
-        JobRenormalizedResultsPersister persister = new JobRenormalizedResultsPersister("foo", Settings.EMPTY, client);
+        JobRenormalizedResultsPersister persister = new JobRenormalizedResultsPersister("foo", client);
 
         ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, 0);
         for (int i=0; i<=JobRenormalizedResultsPersister.BULK_LIMIT; i++) {
@@ -62,7 +61,7 @@ private JobRenormalizedResultsPersister createJobRenormalizedResultsPersister()
         when(bulkResponse.hasFailures()).thenReturn(false);
 
         Client client = new MockClientBuilder("cluster").bulk(bulkResponse).build();
-        return new JobRenormalizedResultsPersister("foo", Settings.EMPTY, client);
+        return new JobRenormalizedResultsPersister("foo", client);
     }
 
     private BucketNormalizable createBucketNormalizable() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
index f2c18ec9d5a4e..e5a9a364a765e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
@@ -61,7 +61,7 @@ public void testPersistBucket_OneRecord() throws IOException {
         AnomalyRecord record = new AnomalyRecord(JOB_ID, new Date(), 600);
         bucket.setRecords(Collections.singletonList(record));
 
-        JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
+        JobResultsPersister persister = new JobResultsPersister(client);
         persister.bulkPersisterBuilder(JOB_ID).persistBucket(bucket).executeRequest();
         BulkRequest bulkRequest = captor.getValue();
         assertEquals(2, bulkRequest.numberOfActions());
@@ -113,7 +113,7 @@ public void testPersistRecords() throws IOException {
         typicals.add(998765.3);
         r1.setTypical(typicals);
 
-        JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
+        JobResultsPersister persister = new JobResultsPersister(client);
         persister.bulkPersisterBuilder(JOB_ID).persistRecords(records).executeRequest();
         BulkRequest bulkRequest = captor.getValue();
         assertEquals(1, bulkRequest.numberOfActions());
@@ -149,7 +149,7 @@ public void testPersistInfluencers() throws IOException {
         inf.setProbability(0.4);
         influencers.add(inf);
 
-        JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
+        JobResultsPersister persister = new JobResultsPersister(client);
         persister.bulkPersisterBuilder(JOB_ID).persistInfluencers(influencers).executeRequest();
         BulkRequest bulkRequest = captor.getValue();
         assertEquals(1, bulkRequest.numberOfActions());
@@ -165,7 +165,7 @@ public void testPersistInfluencers() throws IOException {
     public void testExecuteRequest_ClearsBulkRequest() {
         ArgumentCaptor<BulkRequest> captor = ArgumentCaptor.forClass(BulkRequest.class);
         Client client = mockClient(captor);
-        JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
+        JobResultsPersister persister = new JobResultsPersister(client);
 
         List<Influencer> influencers = new ArrayList<>();
         Influencer inf = new Influencer(JOB_ID, "infName1", "infValue1", new Date(), 600);
@@ -182,7 +182,7 @@ public void testExecuteRequest_ClearsBulkRequest() {
     public void testBulkRequestExecutesWhenReachMaxDocs() {
         ArgumentCaptor<BulkRequest> captor = ArgumentCaptor.forClass(BulkRequest.class);
         Client client = mockClient(captor);
-        JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client);
+        JobResultsPersister persister = new JobResultsPersister(client);
 
         JobResultsPersister.Builder bulkBuilder = persister.bulkPersisterBuilder("foo");
         ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, 0);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
index 4be8d74274cce..5415b46019196 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
@@ -6,9 +6,7 @@
 package org.elasticsearch.xpack.ml.job.process;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -34,14 +32,10 @@ public class DataCountsReporterTests extends ESTestCase {
 
     private Job job;
     private JobDataCountsPersister jobDataCountsPersister;
-    private Settings settings;
     private TimeValue bucketSpan = TimeValue.timeValueSeconds(300);
 
     @Before
     public void setUpMocks() {
-        settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .build();
-
         AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Arrays.asList(new Detector.Builder("metric", "field").build()));
         acBuilder.setBucketSpan(bucketSpan);
         acBuilder.setLatency(TimeValue.ZERO);
@@ -57,8 +51,7 @@ public void setUpMocks() {
     }
 
     public void testSimpleConstructor() throws Exception {
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, new DataCounts(job.getId()),
-                jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
         DataCounts stats = dataCountsReporter.incrementalStats();
         assertNotNull(stats);
         assertAllCountFieldsEqualZero(stats);
@@ -68,8 +61,7 @@ public void testComplexConstructor() throws Exception {
         DataCounts counts = new DataCounts("foo", 1L, 1L, 2L, 0L, 3L, 4L, 5L, 6L, 7L, 8L,
                 new Date(), new Date(), new Date(), new Date(), new Date());
 
-        DataCountsReporter dataCountsReporter =
-                new DataCountsReporter(settings, job, counts, jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, counts, jobDataCountsPersister);
         DataCounts stats = dataCountsReporter.incrementalStats();
         assertNotNull(stats);
         assertAllCountFieldsEqualZero(stats);
@@ -86,8 +78,7 @@ public void testComplexConstructor() throws Exception {
     }
 
     public void testResetIncrementalCounts() throws Exception {
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, new DataCounts(job.getId()),
-                jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
         DataCounts stats = dataCountsReporter.incrementalStats();
         assertNotNull(stats);
         assertAllCountFieldsEqualZero(stats);
@@ -139,16 +130,14 @@ public void testResetIncrementalCounts() throws Exception {
     }
 
     public void testReportLatestTimeIncrementalStats() throws IOException {
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, new DataCounts(job.getId()),
-                jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
         dataCountsReporter.startNewIncrementalCount();
         dataCountsReporter.reportLatestTimeIncrementalStats(5001L);
         assertEquals(5001L, dataCountsReporter.incrementalStats().getLatestRecordTimeStamp().getTime());
     }
 
     public void testReportRecordsWritten() {
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, new DataCounts(job.getId()),
-                jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
         dataCountsReporter.setAnalysedFieldsPerRecord(3);
 
         dataCountsReporter.reportRecordWritten(5, 2000);
@@ -258,8 +247,7 @@ public void testReportRecordsWritten_Given2_000_000Records() {
 
 
     public void testFinishReporting() {
-        DataCountsReporter dataCountsReporter = new DataCountsReporter(settings, job, new DataCounts(job.getId()),
-                jobDataCountsPersister);
+        DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
 
         dataCountsReporter.setAnalysedFieldsPerRecord(3);
         Date now = new Date();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DummyDataCountsReporter.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DummyDataCountsReporter.java
index 98ab4025bffdb..3967e83f5ec56 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DummyDataCountsReporter.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DummyDataCountsReporter.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.ml.job.process;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -27,8 +26,7 @@ class DummyDataCountsReporter extends DataCountsReporter {
     int logStatusCallCount = 0;
 
     DummyDataCountsReporter() {
-        super(Settings.EMPTY, createJob(), new DataCounts("DummyJobId"),
-                mock(JobDataCountsPersister.class));
+        super(createJob(), new DataCounts("DummyJobId"), mock(JobDataCountsPersister.class));
     }
 
     /**
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
index 6d5adeb3fdbf1..18ee9434f0dab 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.ml.job.process.autodetect;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser;
@@ -55,7 +54,7 @@ public void testProcessStartTime() throws Exception {
         try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
                 mock(OutputStream.class), mock(InputStream.class), mock(OutputStream.class),
                 NUMBER_FIELDS, null,
-                new AutodetectResultsParser(Settings.EMPTY), mock(Runnable.class))) {
+                new AutodetectResultsParser(), mock(Runnable.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             ZonedDateTime startTime = process.getProcessStartTime();
@@ -75,7 +74,7 @@ public void testWriteRecord() throws IOException {
         ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
         try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
                 bos, mock(InputStream.class), mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(Settings.EMPTY), mock(Runnable.class))) {
+                new AutodetectResultsParser(), mock(Runnable.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             process.writeRecord(record);
@@ -107,7 +106,7 @@ public void testFlush() throws IOException {
         ByteArrayOutputStream bos = new ByteArrayOutputStream(ControlMsgToProcessWriter.FLUSH_SPACES_LENGTH + 1024);
         try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
                 bos, mock(InputStream.class), mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(Settings.EMPTY), mock(Runnable.class))) {
+                new AutodetectResultsParser(), mock(Runnable.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             FlushJobParams params = FlushJobParams.builder().build();
@@ -137,7 +136,7 @@ public void testWriteMessage(CheckedConsumer<NativeAutodetectProcess> writeFunct
         ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
         try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
                 bos, mock(InputStream.class), mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(Settings.EMPTY), mock(Runnable.class))) {
+                new AutodetectResultsParser(), mock(Runnable.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             writeFunction.accept(process);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
index d2356a79677c3..8a725b95abf9c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.ml.job.process.autodetect.output;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
@@ -238,7 +237,7 @@ public class AutodetectResultsParserTests extends ESTestCase {
 
     public void testParser() throws IOException {
         InputStream inputStream = new ByteArrayInputStream(METRIC_OUTPUT_SAMPLE.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         List<AutodetectResult> results = new ArrayList<>();
         parser.parseResults(inputStream).forEachRemaining(results::add);
         List<Bucket> buckets = results.stream().map(AutodetectResult::getBucket)
@@ -331,7 +330,7 @@ public void testParser() throws IOException {
     @AwaitsFix(bugUrl = "rewrite this test so it doesn't use ~200 lines of json")
     public void testPopulationParser() throws IOException {
         InputStream inputStream = new ByteArrayInputStream(POPULATION_OUTPUT_SAMPLE.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         List<AutodetectResult> results = new ArrayList<>();
         parser.parseResults(inputStream).forEachRemaining(results::add);
         List<Bucket> buckets = results.stream().map(AutodetectResult::getBucket)
@@ -357,7 +356,7 @@ public void testPopulationParser() throws IOException {
     public void testParse_GivenEmptyArray() throws ElasticsearchParseException, IOException {
         String json = "[]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         assertFalse(parser.parseResults(inputStream).hasNext());
     }
 
@@ -365,7 +364,7 @@ public void testParse_GivenModelSizeStats() throws ElasticsearchParseException,
         String json = "[{\"model_size_stats\": {\"job_id\": \"foo\", \"model_bytes\":300}}]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
 
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         List<AutodetectResult> results = new ArrayList<>();
         parser.parseResults(inputStream).forEachRemaining(results::add);
 
@@ -376,7 +375,7 @@ public void testParse_GivenModelSizeStats() throws ElasticsearchParseException,
     public void testParse_GivenCategoryDefinition() throws IOException {
         String json = "[{\"category_definition\": {\"job_id\":\"foo\", \"category_id\":18}}]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         List<AutodetectResult> results = new ArrayList<>();
         parser.parseResults(inputStream).forEachRemaining(results::add);
 
@@ -387,7 +386,7 @@ public void testParse_GivenCategoryDefinition() throws IOException {
     public void testParse_GivenUnknownObject() throws ElasticsearchParseException, IOException {
         String json = "[{\"unknown\":{\"id\": 18}}]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         XContentParseException e = expectThrows(XContentParseException.class,
                 () -> parser.parseResults(inputStream).forEachRemaining(a -> {}));
         assertEquals("[1:3] [autodetect_result] unknown field [unknown], parser not found", e.getMessage());
@@ -396,7 +395,7 @@ public void testParse_GivenUnknownObject() throws ElasticsearchParseException, I
     public void testParse_GivenArrayContainsAnotherArray() throws ElasticsearchParseException, IOException {
         String json = "[[]]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
                 () -> parser.parseResults(inputStream).forEachRemaining(a -> {}));
         assertEquals("unexpected token [START_ARRAY]", e.getMessage());
@@ -411,7 +410,7 @@ public void testParsingExceptionNaN() {
                 + "\"by_field_name\":\"airline\",\"by_field_value\":\"JZA\", \"typical\":[1020.08],\"actual\":[0],"
                 + "\"field_name\":\"responsetime\",\"function\":\"max\",\"partition_field_name\":\"\",\"partition_field_value\":\"\"}]}}]";
         InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser(Settings.EMPTY);
+        AutodetectResultsParser parser = new AutodetectResultsParser();
 
         expectThrows(XContentParseException.class,
                 () -> parser.parseResults(inputStream).forEachRemaining(a -> {}));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandlerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandlerTests.java
index d490d58c3ab52..8c69875a54dbc 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandlerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageHandlerTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.process.logging;
 
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.ESTestCase;
@@ -205,7 +206,7 @@ public void testParseFatalError() throws IOException, IllegalAccessException {
 
     private static void executeLoggingTest(InputStream is, MockLogAppender mockAppender, Level level, String jobId)
             throws IOException {
-        Logger cppMessageLogger = Loggers.getLogger(CppLogMessageHandler.class);
+        Logger cppMessageLogger = LogManager.getLogger(CppLogMessageHandler.class);
         Loggers.addAppender(cppMessageLogger, mockAppender);
 
         Level oldLevel = cppMessageLogger.getLevel();
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
index 804d0c635984d..db47b63015ea6 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
@@ -141,11 +141,11 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
             threadPool.getThreadContext());
 
         Set<Collector> collectors = new HashSet<>();
-        collectors.add(new IndexStatsCollector(settings, clusterService, getLicenseState(), client));
+        collectors.add(new IndexStatsCollector(clusterService, getLicenseState(), client));
         collectors.add(new ClusterStatsCollector(settings, clusterService, getLicenseState(), client, getLicenseService()));
-        collectors.add(new ShardsCollector(settings, clusterService, getLicenseState()));
-        collectors.add(new NodeStatsCollector(settings, clusterService, getLicenseState(), client));
-        collectors.add(new IndexRecoveryCollector(settings, clusterService, getLicenseState(), client));
+        collectors.add(new ShardsCollector(clusterService, getLicenseState()));
+        collectors.add(new NodeStatsCollector(clusterService, getLicenseState(), client));
+        collectors.add(new IndexRecoveryCollector(clusterService, getLicenseState(), client));
         collectors.add(new JobStatsCollector(settings, clusterService, getLicenseState(), client));
         collectors.add(new StatsCollector(settings, clusterService, getLicenseState(), client));
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/Collector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/Collector.java
index b03ce2001b73b..2677e8eb4899f 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/Collector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/Collector.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.XPackField;
@@ -48,9 +47,8 @@ public abstract class Collector extends AbstractComponent {
     protected final ClusterService clusterService;
     protected final XPackLicenseState licenseState;
 
-    public Collector(final Settings settings, final String name, final ClusterService clusterService,
+    public Collector(final String name, final ClusterService clusterService,
                      final Setting<TimeValue> timeoutSetting, final XPackLicenseState licenseState) {
-        super(settings);
         this.name = name;
         this.clusterService = clusterService;
         this.collectionTimeoutSetting = timeoutSetting;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
index b57c1f31ea6bd..bdccb5604a361 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
@@ -36,6 +36,7 @@ public final class StatsCollector extends Collector {
 
     public static final Setting<TimeValue> CCR_STATS_TIMEOUT = collectionTimeoutSetting("ccr.stats.timeout");
 
+    private final Settings settings;
     private final ThreadContext threadContext;
     private final CcrClient ccrClient;
 
@@ -48,12 +49,13 @@ public StatsCollector(
     }
 
     StatsCollector(
-        final Settings settings,
-        final ClusterService clusterService,
-        final XPackLicenseState licenseState,
-        final CcrClient ccrClient,
-        final ThreadContext threadContext) {
-        super(settings, TYPE, clusterService, CCR_STATS_TIMEOUT, licenseState);
+            final Settings settings,
+            final ClusterService clusterService,
+            final XPackLicenseState licenseState,
+            final CcrClient ccrClient,
+            final ThreadContext threadContext) {
+        super(TYPE, clusterService, CCR_STATS_TIMEOUT, licenseState);
+        this.settings = settings;
         this.ccrClient = ccrClient;
         this.threadContext = threadContext;
     }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
index 1a05b03436495..2e9fac0270be5 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
@@ -55,6 +55,7 @@ public class ClusterStatsCollector extends Collector {
      */
     public static final Setting<TimeValue> CLUSTER_STATS_TIMEOUT = collectionTimeoutSetting("cluster.stats.timeout");
 
+    private final Settings settings;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final LicenseService licenseService;
     private final Client client;
@@ -64,7 +65,7 @@ public ClusterStatsCollector(final Settings settings,
                                  final XPackLicenseState licenseState,
                                  final Client client,
                                  final LicenseService licenseService) {
-        this(settings, clusterService, licenseState, client, licenseService, new IndexNameExpressionResolver(Settings.EMPTY));
+        this(settings, clusterService, licenseState, client, licenseService, new IndexNameExpressionResolver(settings));
     }
 
     ClusterStatsCollector(final Settings settings,
@@ -73,8 +74,8 @@ public ClusterStatsCollector(final Settings settings,
                           final Client client,
                           final LicenseService licenseService,
                           final IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClusterStatsMonitoringDoc.TYPE, clusterService, CLUSTER_STATS_TIMEOUT, licenseState);
-
+        super(ClusterStatsMonitoringDoc.TYPE, clusterService, CLUSTER_STATS_TIMEOUT, licenseState);
+        this.settings = settings;
         this.client = client;
         this.licenseService = licenseService;
         this.indexNameExpressionResolver = Objects.requireNonNull(indexNameExpressionResolver);
@@ -112,7 +113,8 @@ protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node,
         // Adds a cluster stats document
         return Collections.singleton(
                 new ClusterStatsMonitoringDoc(clusterUuid, timestamp(), interval, node, clusterName, version,  clusterStats.getStatus(),
-                                              license, apmIndicesExist, xpackUsage, clusterStats, clusterState, clusterNeedsTLSEnabled));
+                                              license, apmIndicesExist, xpackUsage, clusterStats, clusterState,
+                                              clusterNeedsTLSEnabled));
     }
 
     boolean doAPMIndicesExist(final ClusterState clusterState) {
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
index 414945c2a151b..6109902453f17 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
@@ -45,6 +46,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
                                 ClusterState.Metric.NODES));
 
     public static final String TYPE = "cluster_stats";
+    protected static final String SETTING_DISPLAY_NAME = "cluster.metadata.display_name";
 
     private final String clusterName;
     private final String version;
@@ -118,6 +120,14 @@ boolean getClusterNeedsTLSEnabled() {
         return clusterNeedsTLSEnabled;
     }
 
+    String getClusterDisplayName() {
+        MetaData metaData = this.clusterState.getMetaData();
+        if (metaData == null) {
+            return null;
+        }
+        return metaData.settings().get(SETTING_DISPLAY_NAME);
+    }
+
     @Override
     protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("cluster_name", clusterName);
@@ -156,6 +166,23 @@ protected void innerToXContent(XContentBuilder builder, Params params) throws IO
             builder.endObject();
         }
 
+        String displayName = getClusterDisplayName();
+        if (displayName != null) {
+            builder.startObject("cluster_settings");
+            {
+                    builder.startObject("cluster");
+                    {
+                        builder.startObject("metadata");
+                        {
+                            builder.field("display_name", displayName);
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
+            }
+            builder.endObject();
+        }
+
         builder.startObject("stack_stats");
         {
             // in the future, it may be useful to pass in an object that represents APM (and others), but for now this
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollector.java
index ea8b5065387f3..82d2176d415e0 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollector.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -46,12 +45,10 @@ public class IndexRecoveryCollector extends Collector {
 
     private final Client client;
 
-    public IndexRecoveryCollector(final Settings settings,
-                                  final ClusterService clusterService,
+    public IndexRecoveryCollector(final ClusterService clusterService,
                                   final XPackLicenseState licenseState,
                                   final Client client) {
-
-        super(settings, IndexRecoveryMonitoringDoc.TYPE, clusterService, INDEX_RECOVERY_TIMEOUT, licenseState);
+        super(IndexRecoveryMonitoringDoc.TYPE, clusterService, INDEX_RECOVERY_TIMEOUT, licenseState);
         this.client = Objects.requireNonNull(client);
     }
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java
index b6f50ed9e5cfc..7630aa4794b88 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -40,11 +39,10 @@ public class IndexStatsCollector extends Collector {
 
     private final Client client;
 
-    public IndexStatsCollector(final Settings settings,
-                               final ClusterService clusterService,
+    public IndexStatsCollector(final ClusterService clusterService,
                                final XPackLicenseState licenseState,
                                final Client client) {
-        super(settings, "index-stats", clusterService, INDEX_STATS_TIMEOUT, licenseState);
+        super("index-stats", clusterService, INDEX_STATS_TIMEOUT, licenseState);
         this.client = client;
     }
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
index cfbf9b7e0a4b8..8742e0b645f8e 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
@@ -42,6 +42,7 @@ public class JobStatsCollector extends Collector {
      */
     public static final Setting<TimeValue> JOB_STATS_TIMEOUT = collectionTimeoutSetting("ml.job.stats.timeout");
 
+    private final Settings settings;
     private final ThreadContext threadContext;
     private final MachineLearningClient client;
 
@@ -52,7 +53,8 @@ public JobStatsCollector(final Settings settings, final ClusterService clusterSe
 
     JobStatsCollector(final Settings settings, final ClusterService clusterService,
                       final XPackLicenseState licenseState, final MachineLearningClient client, final ThreadContext threadContext) {
-        super(settings, JobStatsMonitoringDoc.TYPE, clusterService, JOB_STATS_TIMEOUT, licenseState);
+        super(JobStatsMonitoringDoc.TYPE, clusterService, JOB_STATS_TIMEOUT, licenseState);
+        this.settings = settings;
         this.client = client;
         this.threadContext = threadContext;
     }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java
index c990485e6a536..bc816cb9d9a05 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -49,12 +48,10 @@ public class NodeStatsCollector extends Collector {
 
     private final Client client;
 
-    public NodeStatsCollector(final Settings settings,
-                              final ClusterService clusterService,
+    public NodeStatsCollector(final ClusterService clusterService,
                               final XPackLicenseState licenseState,
                               final Client client) {
-
-        super(settings, NodeStatsMonitoringDoc.TYPE, clusterService, NODE_STATS_TIMEOUT, licenseState);
+        super(NodeStatsMonitoringDoc.TYPE, clusterService, NODE_STATS_TIMEOUT, licenseState);
         this.client = Objects.requireNonNull(client);
     }
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
index b5a3a2920e2f7..eda782913adb5 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
@@ -30,11 +29,9 @@
  */
 public class ShardsCollector extends Collector {
 
-    public ShardsCollector(final Settings settings,
-                           final ClusterService clusterService,
+    public ShardsCollector(final ClusterService clusterService,
                            final XPackLicenseState licenseState) {
-
-        super(settings, ShardMonitoringDoc.TYPE, clusterService, null, licenseState);
+        super(ShardMonitoringDoc.TYPE, clusterService, null, licenseState);
     }
 
     @Override
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java
index 40c95384d5349..fab40bf0944f1 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java
@@ -37,6 +37,7 @@
 
 public class Exporters extends AbstractLifecycleComponent implements Iterable<Exporter> {
 
+    private final Settings settings;
     private final Map<String, Exporter.Factory> factories;
     private final AtomicReference<Map<String, Exporter>> exporters;
     private final ClusterService clusterService;
@@ -47,7 +48,7 @@ public Exporters(Settings settings, Map<String, Exporter.Factory> factories,
                      ClusterService clusterService, XPackLicenseState licenseState,
                      ThreadContext threadContext) {
         super(settings);
-
+        this.settings = settings;
         this.factories = factories;
         this.exporters = new AtomicReference<>(emptyMap());
         this.threadContext = Objects.requireNonNull(threadContext);
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java
index 97677c3c1c6d5..e035668841f59 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/ClusterAlertHttpResource.java
@@ -11,12 +11,12 @@
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -33,7 +33,7 @@
  */
 public class ClusterAlertHttpResource extends PublishableHttpResource {
 
-    private static final Logger logger = Loggers.getLogger(ClusterAlertHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(ClusterAlertHttpResource.class);
 
     /**
      * Use this to retrieve the version of Cluster Alert in the Watch's JSON response from a request.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
index ded3064a2a66f..c00448c903a47 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java
@@ -7,6 +7,7 @@
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NByteArrayEntity;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -18,7 +19,6 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -39,7 +39,7 @@
  */
 class HttpExportBulk extends ExportBulk {
 
-    private static final Logger logger = Loggers.getLogger(HttpExportBulk.class);
+    private static final Logger logger = LogManager.getLogger(HttpExportBulk.class);
 
     /**
      * The {@link RestClient} managed by the {@link HttpExporter}.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListener.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListener.java
index 09c1f3522b44a..de14e4eeb8134 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListener.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListener.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseListener;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
@@ -23,7 +23,7 @@
  */
 class HttpExportBulkResponseListener implements ResponseListener {
 
-    private static final Logger logger = Loggers.getLogger(HttpExportBulkResponseListener.class);
+    private static final Logger logger = LogManager.getLogger(HttpExportBulkResponseListener.class);
 
     /**
      * Singleton instance.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
index c84d3a99af437..56eba797c3b48 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
@@ -13,6 +13,7 @@
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.RestClient;
@@ -24,7 +25,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -70,7 +70,7 @@
  */
 public class HttpExporter extends Exporter {
 
-    private static final Logger logger = Loggers.getLogger(HttpExporter.class);
+    private static final Logger logger = LogManager.getLogger(HttpExporter.class);
 
     public static final String TYPE = "http";
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/MultiHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/MultiHttpResource.java
index 15747a6f5dbd1..12e2f3f3f1a96 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/MultiHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/MultiHttpResource.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.logging.Loggers;
 
 import java.util.Collections;
 import java.util.List;
@@ -22,7 +22,7 @@
  */
 public class MultiHttpResource extends HttpResource {
 
-    private static final Logger logger = Loggers.getLogger(MultiHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(MultiHttpResource.class);
 
     /**
      * Sub-resources that are grouped to simplify notification.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/NodeFailureListener.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/NodeFailureListener.java
index aa8d2da070eb5..9b597b25c5cc4 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/NodeFailureListener.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/NodeFailureListener.java
@@ -6,13 +6,13 @@
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
 import org.apache.http.HttpHost;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.Node;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.sniff.Sniffer;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 
 /**
  * {@code NodeFailureListener} logs warnings for any node failure, but it can also notify a {@link Sniffer} and/or {@link HttpResource}
@@ -23,7 +23,7 @@
  */
 class NodeFailureListener extends RestClient.FailureListener {
 
-    private static final Logger logger = Loggers.getLogger(NodeFailureListener.class);
+    private static final Logger logger = LogManager.getLogger(NodeFailureListener.class);
 
     /**
      * The optional {@link Sniffer} associated with the {@link RestClient}.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResource.java
index 77f9de409d3bb..78ca43ff9bbf9 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PipelineHttpResource.java
@@ -8,10 +8,10 @@
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
@@ -27,7 +27,7 @@
  */
 public class PipelineHttpResource extends PublishableHttpResource {
 
-    private static final Logger logger = Loggers.getLogger(PipelineHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(PipelineHttpResource.class);
 
     /**
      * The name of the pipeline that is sent to the remote cluster.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
index 21f208a77bf5e..5e6b056959c9d 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
@@ -8,10 +8,10 @@
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
@@ -28,7 +28,7 @@
  */
 public class TemplateHttpResource extends PublishableHttpResource {
 
-    private static final Logger logger = Loggers.getLogger(TemplateHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(TemplateHttpResource.class);
 
     /**
      * The name of the template that is sent to the remote cluster.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
index eec9162e7edb0..252d9072685b6 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/VersionHttpResource.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -12,7 +13,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
@@ -25,7 +25,7 @@
  */
 public class VersionHttpResource extends HttpResource {
 
-    private static final Logger logger = Loggers.getLogger(VersionHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(VersionHttpResource.class);
 
     /**
      * The minimum supported version of Elasticsearch.
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/WatcherExistsHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/WatcherExistsHttpResource.java
index 3ea5b92445fdb..93eed61f4c5ff 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/WatcherExistsHttpResource.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/WatcherExistsHttpResource.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -12,7 +13,6 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -31,7 +31,7 @@
  */
 public class WatcherExistsHttpResource extends PublishableHttpResource {
 
-    private static final Logger logger = Loggers.getLogger(WatcherExistsHttpResource.class);
+    private static final Logger logger = LogManager.getLogger(WatcherExistsHttpResource.class);
     /**
      * Use this to avoid getting any JSON response from a request.
      */
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index a6b0271697334..b2724884a578b 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.monitoring.exporter.local;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -26,7 +27,6 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -80,7 +80,7 @@
 
 public class LocalExporter extends Exporter implements ClusterStateListener, CleanerService.Listener {
 
-    private static final Logger logger = Loggers.getLogger(LocalExporter.class);
+    private static final Logger logger = LogManager.getLogger(LocalExporter.class);
 
     public static final String TYPE = "local";
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 61224ac0fd735..bc531bb2d6b44 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -203,7 +203,12 @@ public void testToXContent() throws IOException {
                                                                 Version.V_6_0_0_beta1);
 
         final ClusterState clusterState = ClusterState.builder(clusterName)
-                                                        .metaData(MetaData.builder().clusterUUID(clusterUuid).build())
+                                                        .metaData(MetaData.builder()
+                                                            .clusterUUID(clusterUuid)
+                                                            .transientSettings(Settings.builder()
+                                                                .put("cluster.metadata.display_name", "my_prod_cluster")
+                                                                .build())
+                                                            .build())
                                                         .stateUUID("_state_uuid")
                                                         .version(12L)
                                                         .nodes(DiscoveryNodes.builder()
@@ -521,6 +526,13 @@ public void testToXContent() throws IOException {
                       + "}"
                     + "}"
                   + "},"
+                  + "\"cluster_settings\":{"
+                    + "\"cluster\":{"
+                      + "\"metadata\":{"
+                        + "\"display_name\":\"my_prod_cluster\""
+                      + "}"
+                    + "}"
+                  + "},"
                   + "\"stack_stats\":{"
                     + "\"apm\":{"
                       + "\"found\":" + apmIndicesExist
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
index f4484aa5ed755..4e251711298a1 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
@@ -55,7 +54,7 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
         final boolean isElectedMaster = randomBoolean();
         whenLocalNodeElectedMaster(isElectedMaster);
 
-        final IndexRecoveryCollector collector = new IndexRecoveryCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexRecoveryCollector collector = new IndexRecoveryCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(isElectedMaster), is(false));
         if (isElectedMaster) {
@@ -65,14 +64,14 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
 
     public void testShouldCollectReturnsFalseIfNotMaster() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
-        final IndexRecoveryCollector collector = new IndexRecoveryCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexRecoveryCollector collector = new IndexRecoveryCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(false), is(false));
     }
 
     public void testShouldCollectReturnsTrue() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
-        final IndexRecoveryCollector collector = new IndexRecoveryCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexRecoveryCollector collector = new IndexRecoveryCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(true), is(true));
         verify(licenseState).isMonitoringAllowed();
@@ -138,7 +137,7 @@ public void testDoCollect() throws Exception {
         final Client client = mock(Client.class);
         when(client.admin()).thenReturn(adminClient);
 
-        final IndexRecoveryCollector collector = new IndexRecoveryCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexRecoveryCollector collector = new IndexRecoveryCollector(clusterService, licenseState, client);
         assertEquals(timeout, collector.getCollectionTimeout());
         assertEquals(recoveryOnly, collector.getActiveRecoveriesOnly());
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java
index ef96726aaee01..c41ce1ec91388 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -50,7 +49,7 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
         final boolean isElectedMaster = randomBoolean();
         whenLocalNodeElectedMaster(isElectedMaster);
 
-        final IndexStatsCollector collector = new IndexStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexStatsCollector collector = new IndexStatsCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(isElectedMaster), is(false));
         if (isElectedMaster) {
@@ -60,14 +59,14 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
 
     public void testShouldCollectReturnsFalseIfNotMaster() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
-        final IndexStatsCollector collector = new IndexStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexStatsCollector collector = new IndexStatsCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(false), is(false));
     }
 
     public void testShouldCollectReturnsTrue() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
-        final IndexStatsCollector collector = new IndexStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexStatsCollector collector = new IndexStatsCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(true), is(true));
         verify(licenseState).isMonitoringAllowed();
@@ -144,7 +143,7 @@ public void testDoCollect() throws Exception {
         final Client client = mock(Client.class);
         when(client.admin()).thenReturn(adminClient);
 
-        final IndexStatsCollector collector = new IndexStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final IndexStatsCollector collector = new IndexStatsCollector(clusterService, licenseState, client);
         assertEquals(timeout, collector.getCollectionTimeout());
 
         final long interval = randomNonNegativeLong();
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java
index 03692cc9d5382..4aca7fcdc60a4 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -43,7 +42,7 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
         final boolean isElectedMaster = randomBoolean();
         whenLocalNodeElectedMaster(isElectedMaster);
 
-        final NodeStatsCollector collector = new NodeStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final NodeStatsCollector collector = new NodeStatsCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(isElectedMaster), is(false));
         if (isElectedMaster) {
@@ -55,7 +54,7 @@ public void testShouldCollectReturnsTrue() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
         final boolean isElectedMaster = true;
 
-        final NodeStatsCollector collector = new NodeStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final NodeStatsCollector collector = new NodeStatsCollector(clusterService, licenseState, client);
 
         assertThat(collector.shouldCollect(isElectedMaster), is(true));
         verify(licenseState).isMonitoringAllowed();
@@ -76,7 +75,7 @@ public void testDoCollectWithFailures() throws Exception {
         final Client client = mock(Client.class);
         thenReturnNodeStats(client, timeout, nodesStatsResponse);
 
-        final NodeStatsCollector collector = new NodeStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final NodeStatsCollector collector = new NodeStatsCollector(clusterService, licenseState, client);
         assertEquals(timeout, collector.getCollectionTimeout());
 
         final FailedNodeException e = expectThrows(FailedNodeException.class, () ->
@@ -110,7 +109,7 @@ public void testDoCollect() throws Exception {
         final Client client = mock(Client.class);
         thenReturnNodeStats(client, timeout, nodesStatsResponse);
 
-        final NodeStatsCollector collector = new NodeStatsCollector(Settings.EMPTY, clusterService, licenseState, client);
+        final NodeStatsCollector collector = new NodeStatsCollector(clusterService, licenseState, client);
         assertEquals(timeout, collector.getCollectionTimeout());
 
         final long interval = randomNonNegativeLong();
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
index 4affc3a164397..83c098002149a 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -48,7 +47,7 @@ public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
         final boolean isElectedMaster = randomBoolean();
         whenLocalNodeElectedMaster(isElectedMaster);
 
-        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, licenseState);
+        final ShardsCollector collector = new ShardsCollector(clusterService, licenseState);
 
         assertThat(collector.shouldCollect(isElectedMaster), is(false));
         if (isElectedMaster) {
@@ -61,7 +60,7 @@ public void testShouldCollectReturnsFalseIfNotMaster() {
         // this controls the blockage
         whenLocalNodeElectedMaster(false);
 
-        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, licenseState);
+        final ShardsCollector collector = new ShardsCollector(clusterService, licenseState);
 
         assertThat(collector.shouldCollect(false), is(false));
     }
@@ -70,14 +69,14 @@ public void testShouldCollectReturnsTrue() {
         when(licenseState.isMonitoringAllowed()).thenReturn(true);
         whenLocalNodeElectedMaster(true);
 
-        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, licenseState);
+        final ShardsCollector collector = new ShardsCollector(clusterService, licenseState);
 
         assertThat(collector.shouldCollect(true), is(true));
         verify(licenseState).isMonitoringAllowed();
     }
 
     public void testDoCollectWhenNoClusterState() throws Exception {
-        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, licenseState);
+        final ShardsCollector collector = new ShardsCollector(clusterService, licenseState);
 
         final Collection<MonitoringDoc> results = collector.doCollect(randomMonitoringNode(random()), randomNonNegativeLong(), null);
         assertThat(results, notNullValue());
@@ -105,7 +104,7 @@ public void testDoCollect() throws Exception {
         when(nodes.get(eq("_current"))).thenReturn(localNode);
         when(clusterState.getNodes()).thenReturn(nodes);
 
-        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, licenseState);
+        final ShardsCollector collector = new ShardsCollector(clusterService, licenseState);
         assertNull(collector.getCollectionTimeout());
         assertArrayEquals(indices, collector.getCollectionIndices());
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
index 07211fd712c5f..037ea03473a49 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
@@ -207,6 +207,12 @@ public void testMonitoringService() throws Exception {
                            .status(),
                    is(RestStatus.CREATED));
 
+        final Settings settings = Settings.builder()
+            .put("cluster.metadata.display_name", "my cluster")
+            .build();
+
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
+
         whenExportersAreReady(() -> {
             final AtomicReference<SearchResponse> searchResponse = new AtomicReference<>();
 
@@ -316,7 +322,7 @@ private void assertMonitoringDocSourceNode(final Map<String, Object> sourceNode)
     private void assertClusterStatsMonitoringDoc(final Map<String, Object> document,
                                                  final boolean apmIndicesExist) {
         final Map<String, Object> source = (Map<String, Object>) document.get("_source");
-        assertEquals(11, source.size());
+        assertEquals(12, source.size());
 
         assertThat((String) source.get("cluster_name"), not(isEmptyOrNullString()));
         assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
@@ -377,6 +383,12 @@ private void assertClusterStatsMonitoringDoc(final Map<String, Object> document,
         assertThat(clusterState.remove("master_node"), notNullValue());
         assertThat(clusterState.remove("nodes"), notNullValue());
         assertThat(clusterState.keySet(), empty());
+
+
+        final Map<String, Object> clusterSettings = (Map<String, Object>) source.get("cluster_settings");
+        assertThat(clusterSettings, notNullValue());
+        assertThat(clusterSettings.remove("cluster"), notNullValue());
+        assertThat(clusterSettings.keySet(), empty());
     }
 
     /**
@@ -617,6 +629,7 @@ public void disableMonitoring() throws Exception {
         final Settings settings = Settings.builder()
                 .putNull("xpack.monitoring.collection.enabled")
                 .putNull("xpack.monitoring.exporters._local.enabled")
+                .putNull("cluster.metadata.display_name")
                 .build();
 
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
index 09b2ccd079a76..62bb18ee03331 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
@@ -195,7 +195,7 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterServic
         }
 
         SchedulerEngine schedulerEngine = new SchedulerEngine(settings, getClock());
-        return Collections.singletonList(new RollupJobTask.RollupJobPersistentTasksExecutor(settings, client, schedulerEngine, threadPool));
+        return Collections.singletonList(new RollupJobTask.RollupJobPersistentTasksExecutor(client, schedulerEngine, threadPool));
     }
 
     // overridable by tests
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
index a38adf5d9de3a..be8f32693a56f 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupResponseTranslator.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.rollup;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.TriFunction;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHits;
@@ -55,7 +55,7 @@
  */
 public class RollupResponseTranslator {
 
-    private static final Logger logger = Loggers.getLogger(RollupResponseTranslator.class);
+    private static final Logger logger = LogManager.getLogger(RollupResponseTranslator.class);
 
     /**
      * Verifies a live-only search response.  Essentially just checks for failure then returns
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
index 778f3d03b69cc..21e76cda58b7c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportGetRollupIndexCapsAction.java
@@ -31,6 +31,7 @@
 public class TransportGetRollupIndexCapsAction extends HandledTransportAction<GetRollupIndexCapsAction.Request,
     GetRollupIndexCapsAction.Response> {
 
+    private final Settings settings;
     private final ClusterService clusterService;
 
     @Inject
@@ -40,6 +41,7 @@ public TransportGetRollupIndexCapsAction(Settings settings, TransportService tra
                                              IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, GetRollupIndexCapsAction.NAME, threadPool, transportService, actionFilters,
             indexNameExpressionResolver, GetRollupIndexCapsAction.Request::new);
+        this.settings = settings;
         this.clusterService = clusterService;
     }
 
@@ -47,7 +49,7 @@ public TransportGetRollupIndexCapsAction(Settings settings, TransportService tra
     protected void doExecute(GetRollupIndexCapsAction.Request request,
                              ActionListener<GetRollupIndexCapsAction.Response> listener) {
 
-        IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(clusterService.getSettings());
+        IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(settings);
         String[] indices = resolver.concreteIndexNames(clusterService.state(),
             request.indicesOptions(), request.indices());
         Map<String, RollableIndexCaps> allCaps = getCapsByRollupIndex(Arrays.asList(indices),
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 903e432036390..45f425ef37951 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.rollup.action;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
@@ -28,7 +29,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -77,12 +77,13 @@
 
 public class TransportRollupSearchAction extends TransportAction<SearchRequest, SearchResponse> {
 
+    private final Settings settings;
     private final Client client;
     private final NamedWriteableRegistry registry;
     private final BigArrays bigArrays;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
-    private static final Logger logger = Loggers.getLogger(RollupSearchAction.class);
+    private static final Logger logger = LogManager.getLogger(RollupSearchAction.class);
 
     @Inject
     public TransportRollupSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
@@ -90,6 +91,7 @@ public TransportRollupSearchAction(Settings settings, ThreadPool threadPool, Tra
                                  Client client, NamedWriteableRegistry registry, BigArrays bigArrays,
                                  ScriptService scriptService, ClusterService clusterService) {
         super(settings, RollupSearchAction.NAME, threadPool, actionFilters, indexNameExpressionResolver, transportService.getTaskManager());
+        this.settings = settings;
         this.client = client;
         this.registry = registry;
         this.bigArrays = bigArrays;
@@ -107,8 +109,9 @@ public TransportRollupSearchAction(Settings settings, ThreadPool threadPool, Tra
 
     @Override
     protected void doExecute(SearchRequest request, ActionListener<SearchResponse> listener) {
-        RollupSearchContext rollupSearchContext = separateIndices(request.indices(),
-                clusterService.state().getMetaData().indices());
+        IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(settings);
+        String[] indices = resolver.concreteIndexNames(clusterService.state(), request.indicesOptions(), request.indices());
+        RollupSearchContext rollupSearchContext = separateIndices(indices, clusterService.state().getMetaData().indices());
 
         MultiSearchRequest msearch = createMSearchRequest(request, registry, rollupSearchContext);
 
@@ -408,9 +411,10 @@ static RollupSearchContext separateIndices(String[] indices, ImmutableOpenMap<St
         });
         assert normal.size() + rollup.size() > 0;
         if (rollup.size() > 1) {
-            throw new IllegalArgumentException("RollupSearch currently only supports searching one rollup index at a time.");
+            throw new IllegalArgumentException("RollupSearch currently only supports searching one rollup index at a time. " +
+                "Found the following rollup indices: " + rollup);
         }
-        return new RollupSearchContext(normal.toArray(new String[normal.size()]), rollup.toArray(new String[rollup.size()]), jobCaps);
+        return new RollupSearchContext(normal.toArray(new String[0]), rollup.toArray(new String[0]), jobCaps);
     }
 
     class TransportHandler implements TransportRequestHandler<SearchRequest> {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
index 3fbe77b64b41c..f545ab049d44d 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
@@ -56,8 +55,8 @@ public static class RollupJobPersistentTasksExecutor extends PersistentTasksExec
         private final SchedulerEngine schedulerEngine;
         private final ThreadPool threadPool;
 
-        public RollupJobPersistentTasksExecutor(Settings settings, Client client, SchedulerEngine schedulerEngine, ThreadPool threadPool) {
-            super(settings, RollupField.TASK_NAME, Rollup.TASK_THREAD_POOL_NAME);
+        public RollupJobPersistentTasksExecutor(Client client, SchedulerEngine schedulerEngine, ThreadPool threadPool) {
+            super(RollupField.TASK_NAME, Rollup.TASK_THREAD_POOL_NAME);
             this.client = client;
             this.schedulerEngine = schedulerEngine;
             this.threadPool = threadPool;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
index abcc0b2c310ac..066233adce29b 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
@@ -687,7 +687,8 @@ public void testTooManyRollups() throws IOException {
         metaMap.put("bar", indexMeta);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> TransportRollupSearchAction.separateIndices(indices, metaMap.build()));
-        assertThat(e.getMessage(), equalTo("RollupSearch currently only supports searching one rollup index at a time."));
+        assertThat(e.getMessage(), equalTo("RollupSearch currently only supports searching one rollup index at a time. " +
+            "Found the following rollup indices: [foo, bar]"));
     }
 
     public void testEmptyMsearch() {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index d8fb442deafde..e9df82fdd5ea4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -107,9 +107,9 @@
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction;
 import org.elasticsearch.xpack.core.security.action.user.DeleteUserAction;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.GetUsersAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
 import org.elasticsearch.xpack.core.security.action.user.SetEnabledAction;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
@@ -125,6 +125,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
@@ -164,9 +165,9 @@
 import org.elasticsearch.xpack.security.action.user.TransportAuthenticateAction;
 import org.elasticsearch.xpack.security.action.user.TransportChangePasswordAction;
 import org.elasticsearch.xpack.security.action.user.TransportDeleteUserAction;
+import org.elasticsearch.xpack.security.action.user.TransportGetUserPrivilegesAction;
 import org.elasticsearch.xpack.security.action.user.TransportGetUsersAction;
 import org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction;
-import org.elasticsearch.xpack.security.action.user.TransportGetUserPrivilegesAction;
 import org.elasticsearch.xpack.security.action.user.TransportPutUserAction;
 import org.elasticsearch.xpack.security.action.user.TransportSetEnabledAction;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
@@ -189,7 +190,6 @@
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
-import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
 import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
@@ -361,7 +361,7 @@ public Collection<Module> createGuiceModules() {
                 b.bind(CompositeRolesStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet
                 b.bind(NativeRoleMappingStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet
                 b.bind(AuditTrailService.class)
-                        .toInstance(new AuditTrailService(settings, Collections.emptyList(), getLicenseState()));
+                    .toInstance(new AuditTrailService(Collections.emptyList(), getLicenseState()));
             });
             return modules;
         }
@@ -440,7 +440,7 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
             }
         }
         final AuditTrailService auditTrailService =
-                new AuditTrailService(settings, new ArrayList<>(auditTrails), getLicenseState());
+                new AuditTrailService(new ArrayList<>(auditTrails), getLicenseState());
         components.add(auditTrailService);
         this.auditTrailService.set(auditTrailService);
 
@@ -512,16 +512,16 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         final Set<RequestInterceptor> requestInterceptors;
         if (XPackSettings.DLS_FLS_ENABLED.get(settings)) {
             requestInterceptors = Collections.unmodifiableSet(Sets.newHashSet(
-                    new SearchRequestInterceptor(settings, threadPool, getLicenseState()),
-                    new UpdateRequestInterceptor(settings, threadPool, getLicenseState()),
-                    new BulkShardRequestInterceptor(settings, threadPool, getLicenseState()),
-                    new ResizeRequestInterceptor(settings, threadPool, getLicenseState(), auditTrailService),
+                    new SearchRequestInterceptor(threadPool, getLicenseState()),
+                    new UpdateRequestInterceptor(threadPool, getLicenseState()),
+                    new BulkShardRequestInterceptor(threadPool, getLicenseState()),
+                    new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService),
                     new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService)));
         } else {
             requestInterceptors = Collections.emptySet();
         }
 
-        securityActionFilter.set(new SecurityActionFilter(settings, authcService.get(), authzService, getLicenseState(),
+        securityActionFilter.set(new SecurityActionFilter(authcService.get(), authzService, getLicenseState(),
                 requestInterceptors, threadPool, securityContext.get(), destructiveOperations));
 
         clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_ADMIN_FILTERED_FIELDS_SETTING,
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
index 3e1f9f97c2fea..8252c7f9a9dce 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
@@ -55,10 +54,9 @@ public class SecurityActionFilter extends AbstractComponent implements ActionFil
     private final SecurityContext securityContext;
     private final DestructiveOperations destructiveOperations;
 
-    public SecurityActionFilter(Settings settings, AuthenticationService authcService, AuthorizationService authzService,
+    public SecurityActionFilter(AuthenticationService authcService, AuthorizationService authzService,
                                 XPackLicenseState licenseState, Set<RequestInterceptor> requestInterceptors, ThreadPool threadPool,
                                 SecurityContext securityContext, DestructiveOperations destructiveOperations) {
-        super(settings);
         this.authcService = authcService;
         this.authzService = authzService;
         this.licenseState = licenseState;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java
index abdaba7cf29d3..5732e187377ee 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
@@ -29,8 +28,7 @@ public class BulkShardRequestInterceptor extends AbstractComponent implements Re
     private final ThreadContext threadContext;
     private final XPackLicenseState licenseState;
 
-    public BulkShardRequestInterceptor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState) {
-        super(settings);
+    public BulkShardRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
         this.threadContext = threadPool.getThreadContext();
         this.licenseState = licenseState;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
index 5f6f4d1643bef..fb8575ce13013 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -25,9 +24,7 @@ abstract class FieldAndDocumentLevelSecurityRequestInterceptor<Request extends I
     private final ThreadContext threadContext;
     private final XPackLicenseState licenseState;
 
-    FieldAndDocumentLevelSecurityRequestInterceptor(Settings settings, ThreadContext threadContext,
-                                                           XPackLicenseState licenseState) {
-        super(settings);
+    FieldAndDocumentLevelSecurityRequestInterceptor(ThreadContext threadContext, XPackLicenseState licenseState) {
         this.threadContext = threadContext;
         this.licenseState = licenseState;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptor.java
index 255f46cb02c2e..b54d9d4335a10 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptor.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
@@ -29,9 +28,8 @@ public final class ResizeRequestInterceptor extends AbstractComponent implements
     private final XPackLicenseState licenseState;
     private final AuditTrailService auditTrailService;
 
-    public ResizeRequestInterceptor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState,
+    public ResizeRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState,
                                     AuditTrailService auditTrailService) {
-        super(settings);
         this.threadContext = threadPool.getThreadContext();
         this.licenseState = licenseState;
         this.auditTrailService = auditTrailService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/SearchRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/SearchRequestInterceptor.java
index 3ceaa02ee7286..5738d3eef5051 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/SearchRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/SearchRequestInterceptor.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -18,8 +17,8 @@
  */
 public class SearchRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor<SearchRequest> {
 
-    public SearchRequestInterceptor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState) {
-        super(settings, threadPool.getThreadContext(), licenseState);
+    public SearchRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
+        super(threadPool.getThreadContext(), licenseState);
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/UpdateRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/UpdateRequestInterceptor.java
index 40b63d943d818..db265333e6965 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/UpdateRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/UpdateRequestInterceptor.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -22,8 +21,8 @@
  */
 public class UpdateRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor<UpdateRequest> {
 
-    public UpdateRequestInterceptor(Settings settings, ThreadPool threadPool, XPackLicenseState licenseState) {
-        super(settings, threadPool.getThreadContext(), licenseState);
+    public UpdateRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
+        super(threadPool.getThreadContext(), licenseState);
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
index d1afab16c8fec..fb5ba96e6dfa4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
@@ -25,6 +25,7 @@
 
 public class TransportChangePasswordAction extends HandledTransportAction<ChangePasswordRequest, ChangePasswordResponse> {
 
+    private final Settings settings;
     private final NativeUsersStore nativeUsersStore;
 
     @Inject
@@ -33,6 +34,7 @@ public TransportChangePasswordAction(Settings settings, ThreadPool threadPool, T
                                          NativeUsersStore nativeUsersStore) {
         super(settings, ChangePasswordAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 ChangePasswordRequest::new);
+        this.settings = settings;
         this.nativeUsersStore = nativeUsersStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
index 4ada1ab0e3331..c56458fd2ea0f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportDeleteUserAction.java
@@ -24,6 +24,7 @@
 
 public class TransportDeleteUserAction extends HandledTransportAction<DeleteUserRequest, DeleteUserResponse> {
 
+    private final Settings settings;
     private final NativeUsersStore usersStore;
 
     @Inject
@@ -32,6 +33,7 @@ public TransportDeleteUserAction(Settings settings, ThreadPool threadPool, Actio
                                      TransportService transportService) {
         super(settings, DeleteUserAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 DeleteUserRequest::new);
+        this.settings = settings;
         this.usersStore = usersStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
index 4a57a918c1af0..61d2fc006e5ed 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
@@ -33,6 +33,7 @@
 
 public class TransportGetUsersAction extends HandledTransportAction<GetUsersRequest, GetUsersResponse> {
 
+    private final Settings settings;
     private final NativeUsersStore usersStore;
     private final ReservedRealm reservedRealm;
 
@@ -42,6 +43,7 @@ public TransportGetUsersAction(Settings settings, ThreadPool threadPool, ActionF
                                    TransportService transportService, ReservedRealm reservedRealm) {
         super(settings, GetUsersAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 GetUsersRequest::new);
+        this.settings = settings;
         this.usersStore = usersStore;
         this.reservedRealm = reservedRealm;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
index 9fd68ea26a299..438dca5c6fb49 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportPutUserAction.java
@@ -31,6 +31,7 @@
 
 public class TransportPutUserAction extends HandledTransportAction<PutUserRequest, PutUserResponse> {
 
+    private final Settings settings;
     private final NativeUsersStore usersStore;
 
     @Inject
@@ -38,6 +39,7 @@ public TransportPutUserAction(Settings settings, ThreadPool threadPool, ActionFi
                                   IndexNameExpressionResolver indexNameExpressionResolver,
                                   NativeUsersStore usersStore, TransportService transportService) {
         super(settings, PutUserAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, PutUserRequest::new);
+        this.settings = settings;
         this.usersStore = usersStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
index a8c884ccc16ae..326e36d9a78e0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledAction.java
@@ -27,6 +27,7 @@
  */
 public class TransportSetEnabledAction extends HandledTransportAction<SetEnabledRequest, SetEnabledResponse> {
 
+    private final Settings settings;
     private final NativeUsersStore usersStore;
 
     @Inject
@@ -35,6 +36,7 @@ public TransportSetEnabledAction(Settings settings, ThreadPool threadPool, Trans
                                         NativeUsersStore usersStore) {
         super(settings, SetEnabledAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 SetEnabledRequest::new);
+        this.settings = settings;
         this.usersStore = usersStore;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
index e36dee3d67c49..aa861ce68576a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.security.audit;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.transport.TransportMessage;
@@ -29,8 +28,7 @@ public String name() {
         return "service";
     }
 
-    public AuditTrailService(Settings settings, List<AuditTrail> auditTrails, XPackLicenseState licenseState) {
-        super(settings);
+    public AuditTrailService(List<AuditTrail> auditTrails, XPackLicenseState licenseState) {
         this.auditTrails = Collections.unmodifiableList(auditTrails);
         this.licenseState = licenseState;
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
index 1dabbbfb28613..6fa9ddb2b8412 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java
@@ -166,6 +166,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
                     TimeValue.timeValueMillis(1L), Property.NodeScope);
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
+    private final Settings settings;
     private final String nodeName;
     private final Client client;
     private final QueueConsumer queueConsumer;
@@ -186,7 +187,7 @@ public String name() {
     }
 
     public IndexAuditTrail(Settings settings, Client client, ThreadPool threadPool, ClusterService clusterService) {
-        super(settings);
+        this.settings = settings;
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java
index 2d948919a84ce..d9de22e83f646 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/DeprecatedLoggingAuditTrail.java
@@ -81,7 +81,6 @@ public DeprecatedLoggingAuditTrail(Settings settings, ClusterService clusterServ
     }
 
     DeprecatedLoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) {
-        super(settings);
         this.logger = logger;
         this.events = parse(LoggingAuditTrail.INCLUDE_EVENT_SETTINGS.get(settings), LoggingAuditTrail.EXCLUDE_EVENT_SETTINGS.get(settings));
         this.includeRequestBody = LoggingAuditTrail.INCLUDE_REQUEST_BODY.get(settings);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
index b38b1ecf24a99..6de4e34f9264d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
@@ -169,7 +169,6 @@ public LoggingAuditTrail(Settings settings, ClusterService clusterService, Threa
     }
 
     LoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) {
-        super(settings);
         this.logger = logger;
         this.events = parse(INCLUDE_EVENT_SETTINGS.get(settings), EXCLUDE_EVENT_SETTINGS.get(settings));
         this.includeRequestBody = INCLUDE_REQUEST_BODY.get(settings);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
index 37832839a61fe..b9c8d542a6004 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java
@@ -64,7 +64,6 @@ public class AuthenticationService extends AbstractComponent {
     public AuthenticationService(Settings settings, Realms realms, AuditTrailService auditTrail,
                                  AuthenticationFailureHandler failureHandler, ThreadPool threadPool,
                                  AnonymousUser anonymousUser, TokenService tokenService) {
-        super(settings);
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.realms = realms;
         this.auditTrail = auditTrail;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
index ce45ee2bedf9b..c090e86fe7ad0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
@@ -43,6 +43,7 @@
  */
 public class Realms extends AbstractComponent implements Iterable<Realm> {
 
+    private final Settings settings;
     private final Environment env;
     private final Map<String, Realm.Factory> factories;
     private final XPackLicenseState licenseState;
@@ -58,7 +59,7 @@ public class Realms extends AbstractComponent implements Iterable<Realm> {
 
     public Realms(Settings settings, Environment env, Map<String, Realm.Factory> factories, XPackLicenseState licenseState,
                   ThreadContext threadContext, ReservedRealm reservedRealm) throws Exception {
-        super(settings);
+        this.settings = settings;
         this.env = env;
         this.factories = factories;
         this.licenseState = licenseState;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index 6002e8aa85baf..389a8874df4f8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -164,6 +164,7 @@ public final class TokenService extends AbstractComponent {
     private static final int MAX_RETRY_ATTEMPTS = 5;
 
     private final SecureRandom secureRandom = new SecureRandom();
+    private final Settings settings;
     private final ClusterService clusterService;
     private final Clock clock;
     private final TimeValue expirationDelay;
@@ -185,7 +186,6 @@ public final class TokenService extends AbstractComponent {
      */
     public TokenService(Settings settings, Clock clock, Client client,
                         SecurityIndexManager securityIndex, ClusterService clusterService) throws GeneralSecurityException {
-        super(settings);
         byte[] saltArr = new byte[SALT_BYTES];
         secureRandom.nextBytes(saltArr);
 
@@ -198,6 +198,7 @@ public TokenService(Settings settings, Clock clock, Client client,
         }
 
         this.clock = clock.withZone(ZoneOffset.UTC);
+        this.settings = settings;
         this.expirationDelay = TOKEN_EXPIRATION.get(settings);
         this.client = client;
         this.securityIndex = securityIndex;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
index 2de7ce8452dde..b30a588f82a34 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
@@ -83,6 +83,8 @@ public class NativeUsersStore extends AbstractComponent {
 
     private final Hasher hasher = Hasher.BCRYPT;
     public static final String RESERVED_USER_TYPE = "reserved-user";
+
+    private final Settings settings;
     private final Client client;
     private final boolean isTribeNode;
     private final ReservedUserInfo disabledDefaultUserInfo;
@@ -91,7 +93,7 @@ public class NativeUsersStore extends AbstractComponent {
     private final SecurityIndexManager securityIndex;
 
     public NativeUsersStore(Settings settings, Client client, SecurityIndexManager securityIndex) {
-        super(settings);
+        this.settings = settings;
         this.client = client;
         this.isTribeNode = XPackClientActionPlugin.isTribeNode(settings);
         this.securityIndex = securityIndex;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMessageBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMessageBuilder.java
index bdc8da0dfca12..af83d26ea8fa2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMessageBuilder.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMessageBuilder.java
@@ -11,9 +11,9 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.logging.Loggers;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.opensaml.saml.saml2.core.Issuer;
@@ -32,7 +32,7 @@ public abstract class SamlMessageBuilder {
     protected final EntityDescriptor identityProvider;
 
     public SamlMessageBuilder(EntityDescriptor identityProvider, SpConfiguration serviceProvider, Clock clock) {
-        this.logger = Loggers.getLogger(getClass());
+        this.logger = LogManager.getLogger(getClass());
         this.identityProvider = identityProvider;
         this.serviceProvider = serviceProvider;
         this.clock = clock;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java
index eacae4621baac..2a77508ac446c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommand.java
@@ -30,6 +30,7 @@
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DelegatedAuthorizationSupport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DelegatedAuthorizationSupport.java
index ff6fc6042e7e8..55266d9143774 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DelegatedAuthorizationSupport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DelegatedAuthorizationSupport.java
@@ -6,10 +6,10 @@
 
 package org.elasticsearch.xpack.security.authc.support;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.LicenseUtils;
@@ -60,7 +60,7 @@ protected DelegatedAuthorizationSupport(Iterable<? extends Realm> allRealms, Lis
         final List<Realm> resolvedLookupRealms = resolveRealms(allRealms, lookupRealms);
         checkForRealmChains(resolvedLookupRealms, settings);
         this.lookup = new RealmUserLookup(resolvedLookupRealms, threadContext);
-        this.logger = Loggers.getLogger(getClass());
+        this.logger = LogManager.getLogger(getClass());
         this.licenseState = licenseState;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
index 5d1ee2b3eb2cf..355957999e944 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
@@ -8,12 +8,12 @@
 import com.unboundid.ldap.sdk.DN;
 import com.unboundid.ldap.sdk.LDAPException;
 import com.unboundid.util.LDAPSDKUsageException;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
@@ -160,7 +160,7 @@ public RealmConfig getRealm() {
      *
      */
     class DistinguishedNamePredicate implements Predicate<FieldExpression.FieldValue> {
-        private static final Logger LOGGER = Loggers.getLogger(DistinguishedNamePredicate.class);
+        private static final Logger LOGGER = LogManager.getLogger(DistinguishedNamePredicate.class);
 
         private final String string;
         private final DN dn;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
index c15c3f1a9866d..ec63bc819f1da 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
@@ -96,13 +96,14 @@ public void onFailure(Exception e) {
         }
     };
 
+    private final Settings settings;
     private final Client client;
     private final boolean isTribeNode;
     private final SecurityIndexManager securityIndex;
     private final List<String> realmsToRefresh = new CopyOnWriteArrayList<>();
 
     public NativeRoleMappingStore(Settings settings, Client client, SecurityIndexManager securityIndex) {
-        super(settings);
+        this.settings = settings;
         this.client = client;
         this.isTribeNode = XPackClientActionPlugin.isTribeNode(settings);
         this.securityIndex = securityIndex;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index c727dbedc03a7..cd4265b18f287 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -108,7 +108,6 @@ public class AuthorizationService extends AbstractComponent {
     public AuthorizationService(Settings settings, CompositeRolesStore rolesStore, ClusterService clusterService,
                                 AuditTrailService auditTrail, AuthenticationFailureHandler authcFailureHandler,
                                 ThreadPool threadPool, AnonymousUser anonymousUser) {
-        super(settings);
         this.rolesStore = rolesStore;
         this.clusterService = clusterService;
         this.auditTrail = auditTrail;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
index 67e21aadcbceb..075a9b889b041 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
@@ -24,6 +24,7 @@
 import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.PERSISTENT_TASK_ORIGIN;
@@ -111,6 +112,7 @@ public static void switchUserBasedOnActionOriginAndExecute(ThreadContext threadC
             case DEPRECATION_ORIGIN:
             case PERSISTENT_TASK_ORIGIN:
             case ROLLUP_ORIGIN:
+            case INDEX_LIFECYCLE_ORIGIN:
                 securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT);
                 break;
             default:
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
index 9af5f3b109ec7..0c3e29500f396 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
@@ -100,7 +100,6 @@ public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, Nat
                                ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
                                List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders,
                                ThreadContext threadContext, XPackLicenseState licenseState) {
-        super(settings);
         this.fileRolesStore = fileRolesStore;
         fileRolesStore.addListener(this::invalidate);
         this.nativeRolesStore = nativeRolesStore;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
index 38055423412dd..fe751d38b0866 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java
@@ -57,6 +57,7 @@ public class FileRolesStore extends AbstractComponent implements BiConsumer<Set<
     private static final Pattern IN_SEGMENT_LINE = Pattern.compile("^\\s+.+");
     private static final Pattern SKIP_LINE = Pattern.compile("(^#.*|^\\s*)");
 
+    private final Settings settings;
     private final Path file;
     private final XPackLicenseState licenseState;
     private final List<Consumer<Set<String>>> listeners = new ArrayList<>();
@@ -70,7 +71,7 @@ public FileRolesStore(Settings settings, Environment env, ResourceWatcherService
 
     FileRolesStore(Settings settings, Environment env, ResourceWatcherService watcherService, Consumer<Set<String>> listener,
                    XPackLicenseState licenseState) throws IOException {
-        super(settings);
+        this.settings = settings;
         this.file = resolveFile(env);
         if (listener != null) {
             listeners.add(listener);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
index a1905db9599d8..66340d0edf35a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
@@ -75,12 +75,13 @@ public class NativePrivilegeStore extends AbstractComponent {
             return a;
         });
 
+    private final Settings settings;
     private final Client client;
     private final SecurityClient securityClient;
     private final SecurityIndexManager securityIndexManager;
 
     public NativePrivilegeStore(Settings settings, Client client, SecurityIndexManager securityIndexManager) {
-        super(settings);
+        this.settings = settings;
         this.client = client;
         this.securityClient = new SecurityClient(client);
         this.securityIndexManager = securityIndexManager;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index a1a76cd872af3..74d1f3b69e230 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -13,6 +13,9 @@
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.MultiSearchResponse;
@@ -50,7 +53,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -88,6 +90,7 @@ public class NativeRolesStore extends AbstractComponent implements BiConsumer<Se
             TimeValue.timeValueMinutes(20), Property.NodeScope, Property.Deprecated);
     private static final String ROLE_DOC_TYPE = "doc";
 
+    private final Settings settings;
     private final Client client;
     private final XPackLicenseState licenseState;
     private final boolean isTribeNode;
@@ -96,7 +99,7 @@ public class NativeRolesStore extends AbstractComponent implements BiConsumer<Se
     private final SecurityIndexManager securityIndex;
 
     public NativeRolesStore(Settings settings, Client client, XPackLicenseState licenseState, SecurityIndexManager securityIndex) {
-        super(settings);
+        this.settings = settings;
         this.client = client;
         this.isTribeNode = XPackClientActionPlugin.isTribeNode(settings);
         this.securityClient = new SecurityClient(client);
@@ -116,33 +119,54 @@ public void getRoleDescriptors(Set<String> names, final ActionListener<RoleRetri
         if (securityIndex.indexExists() == false) {
             // TODO remove this short circuiting and fix tests that fail without this!
             listener.onResponse(RoleRetrievalResult.success(Collections.emptySet()));
-        } else if (names != null && names.size() == 1) {
-            getRoleDescriptor(Objects.requireNonNull(names.iterator().next()), listener);
-        } else {
+        } else if (names == null || names.isEmpty()) {
             securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> {
-                QueryBuilder query;
-                if (names == null || names.isEmpty()) {
-                    query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE);
-                } else {
-                    final String[] roleNames = names.stream().map(NativeRolesStore::getIdForUser).toArray(String[]::new);
-                    query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(ROLE_DOC_TYPE).addIds(roleNames));
-                }
+                QueryBuilder query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE);
                 final Supplier<ThreadContext.StoredContext> supplier = client.threadPool().getThreadContext().newRestorableContext(false);
                 try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) {
                     SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME)
-                            .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
-                            .setQuery(query)
-                            .setSize(1000)
-                            .setFetchSource(true)
-                            .request();
+                        .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
+                        .setQuery(query)
+                        .setSize(1000)
+                        .setFetchSource(true)
+                        .request();
                     request.indicesOptions().ignoreUnavailable();
-                    final ActionListener<Collection<RoleDescriptor>> descriptorsListener = ActionListener.wrap(
-                        roleDescriptors -> listener.onResponse(RoleRetrievalResult.success(new HashSet<>(roleDescriptors))),
-                        e -> listener.onResponse(RoleRetrievalResult.failure(e)));
-                    ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, descriptorsListener),
-                            (hit) -> transformRole(hit.getId(), hit.getSourceRef(), logger, licenseState));
+                    ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier,
+                            ActionListener.wrap(roles -> listener.onResponse(RoleRetrievalResult.success(new HashSet<>(roles))),
+                                e -> listener.onResponse(RoleRetrievalResult.failure(e)))),
+                        (hit) -> transformRole(hit.getId(), hit.getSourceRef(), logger, licenseState));
                 }
             });
+        } else if (names.size() == 1) {
+            getRoleDescriptor(Objects.requireNonNull(names.iterator().next()), listener);
+        } else {
+            securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> {
+                final String[] roleIds = names.stream().map(NativeRolesStore::getIdForRole).toArray(String[]::new);
+                MultiGetRequest multiGetRequest = client.prepareMultiGet().add(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, roleIds).request();
+                executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, multiGetRequest,
+                    ActionListener.<MultiGetResponse>wrap(mGetResponse -> {
+                            final MultiGetItemResponse[] responses = mGetResponse.getResponses();
+                            Set<RoleDescriptor> descriptors = new HashSet<>();
+                            for (int i = 0; i < responses.length; i++) {
+                                MultiGetItemResponse item = responses[i];
+                                if (item.isFailed()) {
+                                    final Exception failure = item.getFailure().getFailure();
+                                    for (int j = i + 1; j < responses.length; j++) {
+                                        item = responses[j];
+                                        if (item.isFailed()) {
+                                            failure.addSuppressed(failure);
+                                        }
+                                    }
+                                    listener.onResponse(RoleRetrievalResult.failure(failure));
+                                    return;
+                                } else if (item.getResponse().isExists()) {
+                                    descriptors.add(transformRole(item.getResponse()));
+                                }
+                            }
+                            listener.onResponse(RoleRetrievalResult.success(descriptors));
+                        },
+                        e -> listener.onResponse(RoleRetrievalResult.failure(e))), client::multiGet);
+            });
         }
     }
 
@@ -157,7 +181,7 @@ public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionLi
         } else {
             securityIndex.checkIndexVersionThenExecute(listener::onFailure, () -> {
                 DeleteRequest request = client.prepareDelete(SecurityIndexManager.SECURITY_INDEX_NAME,
-                    ROLE_DOC_TYPE, getIdForUser(deleteRoleRequest.name())).request();
+                    ROLE_DOC_TYPE, getIdForRole(deleteRoleRequest.name())).request();
                 request.setRefreshPolicy(deleteRoleRequest.getRefreshPolicy());
                 executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request,
                     new ActionListener<DeleteResponse>() {
@@ -199,7 +223,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final
                 listener.onFailure(e);
                 return;
             }
-            final IndexRequest indexRequest = client.prepareIndex(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role.getName()))
+            final IndexRequest indexRequest = client.prepareIndex(SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForRole(role.getName()))
                     .setSource(xContentBuilder)
                     .setRefreshPolicy(request.getRefreshPolicy())
                     .request();
@@ -315,7 +339,7 @@ private void executeGetRoleRequest(String role, ActionListener<GetResponse> list
         securityIndex.checkIndexVersionThenExecute(listener::onFailure, () ->
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
                     client.prepareGet(SECURITY_INDEX_NAME,
-                            ROLE_DOC_TYPE, getIdForUser(role)).request(),
+                            ROLE_DOC_TYPE, getIdForRole(role)).request(),
                     listener,
                     client::get));
     }
@@ -395,7 +419,7 @@ public static void addSettings(List<Setting<?>> settings) {
     /**
      * Gets the document's id field for the given role name.
      */
-    private static String getIdForUser(final String roleName) {
+    private static String getIdForRole(final String roleName) {
         return ROLE_TYPE + "-" + roleName;
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
index dd1e387b98989..c2066996f9ce4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/SecurityBaseRestHandler.java
@@ -24,6 +24,7 @@
  */
 public abstract class SecurityBaseRestHandler extends BaseRestHandler {
 
+    private final Settings settings;
     protected final XPackLicenseState licenseState;
 
     /**
@@ -32,6 +33,7 @@ public abstract class SecurityBaseRestHandler extends BaseRestHandler {
      */
     protected SecurityBaseRestHandler(Settings settings, XPackLicenseState licenseState) {
         super(settings);
+        this.settings = settings;
         this.licenseState = licenseState;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
index 1b64b3ce2baee..9907e3345232e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
@@ -34,12 +34,13 @@
 public class RestChangePasswordAction extends SecurityBaseRestHandler implements RestRequestFilter {
 
     private final SecurityContext securityContext;
-    private final Hasher passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
+    private final Hasher passwordHasher;
 
     public RestChangePasswordAction(Settings settings, RestController controller, SecurityContext securityContext,
                                     XPackLicenseState licenseState) {
         super(settings, licenseState);
         this.securityContext = securityContext;
+        passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
         controller.registerHandler(POST, "/_xpack/security/user/{username}/_password", this);
         controller.registerHandler(PUT, "/_xpack/security/user/{username}/_password", this);
         controller.registerHandler(POST, "/_xpack/security/user/_password", this);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
index 3a4cb122eb495..6373d4a25c350 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
@@ -41,10 +41,11 @@ public class RestPutUserAction extends SecurityBaseRestHandler implements RestRe
     private static final Logger logger = LogManager.getLogger(RestPutUserAction.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
-    private final Hasher passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
+    private final Hasher passwordHasher;
 
     public RestPutUserAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
         super(settings, licenseState);
+        passwordHasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
         controller.registerHandler(POST, "/_xpack/security/user/{username}", this);
         controller.registerHandler(PUT, "/_xpack/security/user/{username}", this);
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
index 29c81be732c19..b59e656c5b92d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@@ -83,7 +83,6 @@ public SecurityServerTransportInterceptor(Settings settings,
                                               SecurityContext securityContext,
                                               DestructiveOperations destructiveOperations,
                                               ClusterService clusterService) {
-        super(settings);
         this.settings = settings;
         this.threadPool = threadPool;
         this.authcService = authcService;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
index 0bbd83efaed17..cd611baea1455 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
@@ -8,6 +8,7 @@
 import io.netty.channel.Channel;
 import io.netty.handler.ssl.SslHandler;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
@@ -17,7 +18,6 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.open.OpenIndexAction;
 import org.elasticsearch.action.support.DestructiveOperations;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.transport.TaskTransportChannel;
 import org.elasticsearch.transport.TcpTransportChannel;
@@ -67,7 +67,7 @@ void inbound(String action, TransportRequest request, TransportChannel transport
      * request is properly authenticated and authorized
      */
     class NodeProfile implements ServerTransportFilter {
-        private static final Logger logger = Loggers.getLogger(NodeProfile.class);
+        private static final Logger logger = LogManager.getLogger(NodeProfile.class);
 
         private final AuthenticationService authcService;
         private final AuthorizationService authzService;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
index 95d821cb256e9..f6e5552ddbc53 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/TemplateUpgraderTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.TemplateUpgradeService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.SecurityIntegTestCase;
@@ -55,7 +54,7 @@ public void testTemplatesWorkAsExpected() throws Exception {
         assertAcked(putIndexTemplateResponse);
         assertTemplates("removed-template", "added-template");
 
-        TemplateUpgradeService templateUpgradeService = new TemplateUpgradeService(Settings.EMPTY, client, clusterService, threadPool,
+        TemplateUpgradeService templateUpgradeService = new TemplateUpgradeService(client, clusterService, threadPool,
                 Collections.singleton(indexTemplateMetaDataUpgraders));
 
         // ensure the cluster listener gets triggered
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
index 44a48f089ade9..46a80c26a9bbf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java
@@ -83,7 +83,7 @@ public void init() throws Exception {
         when(state.nodes()).thenReturn(nodes);
 
         SecurityContext securityContext = new SecurityContext(settings, threadContext);
-        filter = new SecurityActionFilter(Settings.EMPTY, authcService, authzService,
+        filter = new SecurityActionFilter(authcService, authzService,
                         licenseState, new HashSet<>(), threadPool, securityContext, destructiveOperations);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
index a5798be9746ed..bf7ab102c2a21 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java
@@ -40,7 +40,7 @@ public void testInterceptorThrowsWhenFLSDLSEnabled() {
         when(licenseState.isAuditingAllowed()).thenReturn(true);
         when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        AuditTrailService auditTrailService = new AuditTrailService(Settings.EMPTY, Collections.emptyList(), licenseState);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         Authentication authentication = new Authentication(new User("john", "role"), new RealmRef(null, null, null),
                 new RealmRef(null, null, null));
         final FieldPermissions fieldPermissions;
@@ -87,7 +87,7 @@ public void testInterceptorThrowsWhenTargetHasGreaterPermissions() throws Except
         when(licenseState.isAuditingAllowed()).thenReturn(true);
         when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        AuditTrailService auditTrailService = new AuditTrailService(Settings.EMPTY, Collections.emptyList(), licenseState);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         Authentication authentication = new Authentication(new User("john", "role"), new RealmRef(null, null, null),
                 new RealmRef(null, null, null));
         Role role = Role.builder()
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
index 008928794db46..e956ad5e03151 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java
@@ -44,7 +44,7 @@ public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() {
         ThreadPool threadPool = mock(ThreadPool.class);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
-        AuditTrailService auditTrailService = new AuditTrailService(Settings.EMPTY, Collections.emptyList(), licenseState);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         final Authentication authentication = new Authentication(new User("john", "role"), new RealmRef(null, null, null), null);
         final FieldPermissions fieldPermissions;
         final boolean useFls = randomBoolean();
@@ -67,7 +67,7 @@ public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() {
         threadContext.putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, accessControl);
 
         ResizeRequestInterceptor resizeRequestInterceptor =
-                new ResizeRequestInterceptor(Settings.EMPTY, threadPool, licenseState, auditTrailService);
+                new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
 
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
                 () -> resizeRequestInterceptor.intercept(new ResizeRequest("bar", "foo"), authentication, role, action));
@@ -84,7 +84,7 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions()
         ThreadPool threadPool = mock(ThreadPool.class);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
-        AuditTrailService auditTrailService = new AuditTrailService(Settings.EMPTY, Collections.emptyList(), licenseState);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         final Authentication authentication = new Authentication(new User("john", "role"), new RealmRef(null, null, null), null);
         Role role = Role.builder()
                 .add(IndexPrivilege.ALL, "target")
@@ -94,7 +94,7 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions()
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.emptyMap());
         threadContext.putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, accessControl);
         ResizeRequestInterceptor resizeRequestInterceptor =
-                new ResizeRequestInterceptor(Settings.EMPTY, threadPool, licenseState, auditTrailService);
+                new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
                 () -> resizeRequestInterceptor.intercept(new ResizeRequest("target", "source"), authentication, role, action));
         assertEquals("Resizing an index is not allowed when the target index has more permissions than the source index",
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
index 13a7e5c3cf71a..6ce5ad0ccfab2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.security.audit;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -46,7 +45,7 @@ public void init() throws Exception {
         }
         auditTrails = unmodifiableList(auditTrailsBuilder);
         licenseState = mock(XPackLicenseState.class);
-        service = new AuditTrailService(Settings.EMPTY, auditTrails, licenseState);
+        service = new AuditTrailService(auditTrails, licenseState);
         isAuditingAllowed = randomBoolean();
         when(licenseState.isAuditingAllowed()).thenReturn(isAuditingAllowed);
         token = mock(AuthenticationToken.class);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
index 22071d6010d08..8d10f3ffb6946 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.security.authc.saml;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.xml.security.Init;
 import org.apache.xml.security.encryption.EncryptedData;
 import org.apache.xml.security.encryption.EncryptedKey;
@@ -16,7 +17,6 @@
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.hamcrest.Matchers;
@@ -57,7 +57,6 @@
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
-
 import java.io.IOException;
 import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
@@ -125,7 +124,7 @@ public class SamlAuthenticatorTests extends SamlTestCase {
     public static void init() throws Exception {
         assumeFalse("Can't run in a FIPS JVM, there is no DOM XMLSignature Factory so we can't sign XML documents", inFipsJvm());
         // TODO: Refactor the signing to use org.opensaml.xmlsec.signature.support.Signer so that we can run the tests
-        SamlUtils.initialize(Loggers.getLogger(SamlAuthenticatorTests.class));
+        SamlUtils.initialize(LogManager.getLogger(SamlAuthenticatorTests.class));
         // Initialise Apache XML security so that the signDoc methods work correctly.
         Init.init();
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlTestCase.java
index adaba34a73aa3..7bf13e8be265c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlTestCase.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.security.authc.saml;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
 import org.elasticsearch.xpack.core.ssl.PemUtils;
@@ -37,7 +37,7 @@ public abstract class SamlTestCase extends ESTestCase {
 
     @BeforeClass
     public static void setupSaml() throws Exception {
-        Logger logger = Loggers.getLogger(SamlTestCase.class);
+        Logger logger = LogManager.getLogger(SamlTestCase.class);
         if (isTurkishLocale()) {
             // See: https://github.com/elastic/x-pack-elasticsearch/issues/2815
             logger.warn("Attempting to run SAML test on turkish-like locale, but that breaks OpenSAML. Switching to English.");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
index 9c9f2b1b1a42a..66b1e9d9c2ad4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
@@ -140,7 +140,7 @@ public void testSwitchAndExecuteXpackUser() throws Exception {
         threadContext.putHeader(headerName, headerValue);
         threadContext.putTransient(ClientHelper.ACTION_ORIGIN_TRANSIENT_NAME,
                 randomFrom(ClientHelper.ML_ORIGIN, ClientHelper.WATCHER_ORIGIN, ClientHelper.DEPRECATION_ORIGIN,
-                        ClientHelper.MONITORING_ORIGIN, ClientHelper.PERSISTENT_TASK_ORIGIN));
+                        ClientHelper.MONITORING_ORIGIN, ClientHelper.PERSISTENT_TASK_ORIGIN, ClientHelper.INDEX_LIFECYCLE_ORIGIN));
 
         AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(threadContext, securityContext, consumer);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
index ee40d3e24bb8a..f17e9f4ff0871 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
@@ -57,7 +57,7 @@ public void init() throws Exception {
                 IPFilter.PROFILE_FILTER_DENY_SETTING)));
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isIpFilteringAllowed()).thenReturn(true);
-        AuditTrailService auditTrailService = new AuditTrailService(settings, Collections.emptyList(), licenseState);
+        AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         IPFilter ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         if (isHttpEnabled) {
diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle
index 62097e76b97ea..3cde541d304d5 100644
--- a/x-pack/plugin/sql/build.gradle
+++ b/x-pack/plugin/sql/build.gradle
@@ -51,6 +51,16 @@ bundlePlugin {
     }
 }
 
+// add all sub-projects of the qa sub-project
+gradle.projectsEvaluated {
+    project.subprojects
+            .find { it.path == project.path + ":qa" }
+            .subprojects
+            .findAll { it.path.startsWith(project.path + ":qa") }
+            .each { check.dependsOn it.check }
+}
+
+
 /**********************************************
  *          SQL Parser regeneration           *
  **********************************************/
diff --git a/x-pack/qa/sql/build.gradle b/x-pack/plugin/sql/qa/build.gradle
similarity index 97%
rename from x-pack/qa/sql/build.gradle
rename to x-pack/plugin/sql/qa/build.gradle
index baaf0451e51f2..6b55b6363f2ca 100644
--- a/x-pack/qa/sql/build.gradle
+++ b/x-pack/plugin/sql/qa/build.gradle
@@ -16,7 +16,7 @@ dependencies {
 
   // CLI testing dependencies
   compile project(path: xpackModule('sql:sql-cli'), configuration: 'nodeps')
-  compile "org.jline:jline:3.6.0"
+  compile "org.jline:jline:3.8.2"
 }
 
 /* disable unit tests because these are all integration tests used
@@ -82,7 +82,7 @@ subprojects {
      * dependencies but we don't really want them because they cause
      * all kinds of trouble with the jar hell checks. So we suppress
      * them explicitly for non-es projects. */
-    testCompile(xpackProject('qa:sql')) {
+    testCompile(xpackProject('plugin:sql:qa')) {
       transitive = false
     }
     testCompile "org.elasticsearch.test:framework:${version}"
@@ -104,7 +104,7 @@ subprojects {
     testRuntime (xpackProject('plugin:sql:sql-action')) {
       transitive = false
     }
-    testRuntime "org.jline:jline:3.6.0"
+    testRuntime "org.jline:jline:3.8.2"
   }
 
   if (project.name != 'security') {
diff --git a/x-pack/qa/sql/multinode/build.gradle b/x-pack/plugin/sql/qa/multi-node/build.gradle
similarity index 100%
rename from x-pack/qa/sql/multinode/build.gradle
rename to x-pack/plugin/sql/qa/multi-node/build.gradle
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliSelectIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliSelectIT.java
similarity index 73%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliSelectIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliSelectIT.java
index 56a112df02135..9e5c78219100e 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliSelectIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliSelectIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.SelectTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.SelectTestCase;
 
 public class CliSelectIT extends SelectTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliShowIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliShowIT.java
similarity index 73%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliShowIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliShowIT.java
index 2720b45d2a72e..0b5e97bf534b5 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliShowIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/CliShowIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.ShowTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.ShowTestCase;
 
 public class CliShowIT extends ShowTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDatabaseMetaDataIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcDatabaseMetaDataIT.java
similarity index 73%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDatabaseMetaDataIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcDatabaseMetaDataIT.java
index f653049b9a1ae..344ae01f5b9cb 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDatabaseMetaDataIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcDatabaseMetaDataIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.DatabaseMetaDataTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.DatabaseMetaDataTestCase;
 
 public class JdbcDatabaseMetaDataIT extends DatabaseMetaDataTestCase {
 }
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcErrorsIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcErrorsIT.java
similarity index 72%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcErrorsIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcErrorsIT.java
index 946a9b6c73165..fec89fac11f1c 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcErrorsIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcErrorsIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ErrorsTestCase;
 
 public class JdbcErrorsIT extends ErrorsTestCase {
 }
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcPreparedStatementIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcPreparedStatementIT.java
similarity index 73%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcPreparedStatementIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcPreparedStatementIT.java
index 155e9bf161b1c..b06746acb79ce 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcPreparedStatementIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcPreparedStatementIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.PreparedStatementTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.PreparedStatementTestCase;
 
 public class JdbcPreparedStatementIT extends PreparedStatementTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShowTablesIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcShowTablesIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShowTablesIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcShowTablesIT.java
index f68b022f6adae..0936dfabf2bf8 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShowTablesIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/JdbcShowTablesIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ShowTablesTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ShowTablesTestCase;
 
 public class JdbcShowTablesIT extends ShowTablesTestCase {
 }
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlIT.java
similarity index 79%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlIT.java
index 231cee1f343bb..ad81bc380d3d1 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
-import org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase;
+import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase;
 
 /**
  * Integration test for the rest sql action. The one that speaks json directly to a
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
similarity index 96%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java
rename to x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
index 75eaefed77925..9df646872b1e8 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/RestSqlMultinodeIT.java
+++ b/x-pack/plugin/sql/qa/multi-node/src/test/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.multi_node;
 
 import org.apache.http.HttpHost;
 import org.elasticsearch.client.Request;
@@ -25,8 +25,8 @@
 import java.util.Map;
 
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.columnInfo;
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.randomMode;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.randomMode;
 
 /**
  * Tests specific to multiple nodes.
diff --git a/x-pack/qa/sql/security/build.gradle b/x-pack/plugin/sql/qa/security/build.gradle
similarity index 100%
rename from x-pack/qa/sql/security/build.gradle
rename to x-pack/plugin/sql/qa/security/build.gradle
diff --git a/x-pack/qa/sql/security/roles.yml b/x-pack/plugin/sql/qa/security/roles.yml
similarity index 100%
rename from x-pack/qa/sql/security/roles.yml
rename to x-pack/plugin/sql/qa/security/roles.yml
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliErrorsIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliErrorsIT.java
similarity index 80%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliErrorsIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliErrorsIT.java
index 92d9608a527f1..a37be9455705a 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliErrorsIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliErrorsIT.java
@@ -3,11 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
 
 public class CliErrorsIT extends ErrorsTestCase {
     @Override
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliFetchSizeIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliFetchSizeIT.java
similarity index 80%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliFetchSizeIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliFetchSizeIT.java
index c8ca7db71d1e2..f694d4c9118c4 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliFetchSizeIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliFetchSizeIT.java
@@ -3,11 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.cli.FetchSizeTestCase;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.FetchSizeTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
 
 public class CliFetchSizeIT extends FetchSizeTestCase {
     @Override
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java
similarity index 96%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java
index 3e0b578913841..2995cc4e57750 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java
@@ -3,13 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
-import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.ErrorsTestCase;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
@@ -20,7 +20,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase.elasticsearchAddress;
+import static org.elasticsearch.xpack.sql.qa.cli.CliIntegrationTestCase.elasticsearchAddress;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.startsWith;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSelectIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSelectIT.java
similarity index 80%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSelectIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSelectIT.java
index 596fd1e723644..93faf2d70e937 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSelectIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSelectIT.java
@@ -3,11 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
-import org.elasticsearch.xpack.qa.sql.cli.SelectTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.SelectTestCase;
 
 public class CliSelectIT extends SelectTestCase {
     @Override
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliShowIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliShowIT.java
similarity index 80%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliShowIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliShowIT.java
index c05dbcc3d1369..f527188371e84 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliShowIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliShowIT.java
@@ -3,11 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
-import org.elasticsearch.xpack.qa.sql.cli.ShowTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.ShowTestCase;
 
 public class CliShowIT extends ShowTestCase {
     @Override
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcConnectionIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcConnectionIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcConnectionIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcConnectionIT.java
index 08aa73f68b9ea..8a4815a4e1eaf 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcConnectionIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcConnectionIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.ConnectionTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ConnectionTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcCsvSpecIT.java
similarity index 85%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcCsvSpecIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcCsvSpecIT.java
index e5fdf0baf452c..ada9c85b1c1e0 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcCsvSpecIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcCsvSpecIT.java
@@ -3,11 +3,11 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvSpecTestCase;
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcDatabaseMetaDataIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcDatabaseMetaDataIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcDatabaseMetaDataIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcDatabaseMetaDataIT.java
index f4aafe4090b95..fabf2489bd8ac 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcDatabaseMetaDataIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcDatabaseMetaDataIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.DatabaseMetaDataTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.DatabaseMetaDataTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcErrorsIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcErrorsIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcErrorsIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcErrorsIT.java
index 2ed8ac7941ff0..d6a3a55733462 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcErrorsIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcErrorsIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ErrorsTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcFetchSizeIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcFetchSizeIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcFetchSizeIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcFetchSizeIT.java
index ac239193e9938..c75bbc1a50ac0 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcFetchSizeIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcFetchSizeIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.FetchSizeTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.FetchSizeTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcPreparedStatementIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcPreparedStatementIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcPreparedStatementIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcPreparedStatementIT.java
index 3ecb0d388c204..36a65f88a6f2d 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcPreparedStatementIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcPreparedStatementIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.PreparedStatementTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.PreparedStatementTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java
similarity index 97%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java
index 848b98eeb7bf8..d47b06289a8b3 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSecurityIT.java
@@ -3,12 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.xpack.qa.sql.jdbc.LocalH2;
+import org.elasticsearch.xpack.sql.qa.jdbc.LocalH2;
 
 import java.net.URISyntaxException;
 import java.nio.file.Files;
@@ -24,10 +24,10 @@
 import java.util.Map;
 import java.util.Properties;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert.assertResultSets;
-import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase.elasticsearchAddress;
-import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase.randomKnownTimeZone;
-import static org.elasticsearch.xpack.qa.sql.security.RestSqlIT.SSL_ENABLED;
+import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcAssert.assertResultSets;
+import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase.elasticsearchAddress;
+import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase.randomKnownTimeZone;
+import static org.elasticsearch.xpack.sql.qa.security.RestSqlIT.SSL_ENABLED;
 import static org.hamcrest.Matchers.containsString;
 
 public class JdbcSecurityIT extends SqlSecurityTestCase {
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcShowTablesIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcShowTablesIT.java
similarity index 87%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcShowTablesIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcShowTablesIT.java
index ab76b3f33a132..ba54795336f5f 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcShowTablesIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcShowTablesIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.ShowTablesTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ShowTablesTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSimpleExampleIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSimpleExampleIT.java
similarity index 88%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSimpleExampleIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSimpleExampleIT.java
index b01fe72333b64..d7aeff61947c6 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSimpleExampleIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSimpleExampleIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.SimpleExampleTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.SimpleExampleTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSqlSpecIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSqlSpecIT.java
similarity index 90%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSqlSpecIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSqlSpecIT.java
index 609847f513e3a..c8ee720c39f67 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSqlSpecIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/JdbcSqlSpecIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.qa.sql.jdbc.SqlSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.SqlSpecTestCase;
 
 import java.util.Properties;
 
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlIT.java
similarity index 93%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlIT.java
index e5408e48dac02..362b3a93211ea 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlIT.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.io.PathUtils;
@@ -11,20 +11,20 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase;
 
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
 /**
  * Integration test for the rest sql action. The one that speaks json directly to a
  * user rather than to the JDBC driver or CLI.
  */
 public class RestSqlIT extends RestSqlTestCase {
-    static final boolean SSL_ENABLED = Booleans.parseBoolean(System.getProperty("tests.ssl.enabled"));
+    static final boolean SSL_ENABLED = Booleans.parseBoolean(System.getProperty("tests.ssl.enabled"), false);
 
     static Settings securitySettings() {
         String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
similarity index 98%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
index a7fdaf7e4743c..ac3445f23b666 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
@@ -29,8 +29,8 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.columnInfo;
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.randomMode;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.randomMode;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
similarity index 99%
rename from x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java
rename to x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
index 0e3d2cab2ff84..848fee17c7f1f 100644
--- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java
+++ b/x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.security;
+package org.elasticsearch.xpack.sql.qa.security;
 
 import org.apache.lucene.util.SuppressForbidden;
 import org.elasticsearch.ElasticsearchParseException;
diff --git a/x-pack/qa/sql/security/src/test/resources/plugin-security.policy b/x-pack/plugin/sql/qa/security/src/test/resources/plugin-security.policy
similarity index 100%
rename from x-pack/qa/sql/security/src/test/resources/plugin-security.policy
rename to x-pack/plugin/sql/qa/security/src/test/resources/plugin-security.policy
diff --git a/x-pack/qa/sql/security/ssl/build.gradle b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
similarity index 100%
rename from x-pack/qa/sql/security/ssl/build.gradle
rename to x-pack/plugin/sql/qa/security/with-ssl/build.gradle
diff --git a/x-pack/qa/sql/security/no-ssl/build.gradle b/x-pack/plugin/sql/qa/security/without-ssl/build.gradle
similarity index 100%
rename from x-pack/qa/sql/security/no-ssl/build.gradle
rename to x-pack/plugin/sql/qa/security/without-ssl/build.gradle
diff --git a/x-pack/qa/sql/no-security/build.gradle b/x-pack/plugin/sql/qa/single-node/build.gradle
similarity index 100%
rename from x-pack/qa/sql/no-security/build.gradle
rename to x-pack/plugin/sql/qa/single-node/build.gradle
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliErrorsIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliErrorsIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliErrorsIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliErrorsIT.java
index be6c003c7460c..d8817d23efe57 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliErrorsIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliErrorsIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.ErrorsTestCase;
 
 public class CliErrorsIT extends ErrorsTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java
similarity index 98%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java
index cbf6d0d476e57..4296c5ae06919 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.CliIntegrationTestCase;
 
 import java.io.IOException;
 
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliFetchSizeIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliFetchSizeIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliFetchSizeIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliFetchSizeIT.java
index e4d2ef1a0e2ca..eb1e8d32a6bf7 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliFetchSizeIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliFetchSizeIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.FetchSizeTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.FetchSizeTestCase;
 
 public class CliFetchSizeIT extends FetchSizeTestCase {
 }
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliSelectIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliSelectIT.java
similarity index 72%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliSelectIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliSelectIT.java
index af6f986e1bc46..76c2a324ed836 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliSelectIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliSelectIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.SelectTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.SelectTestCase;
 
 public class CliSelectIT extends SelectTestCase {
 }
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliShowIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliShowIT.java
similarity index 73%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliShowIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliShowIT.java
index 07e544094d547..d3557d1afa434 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/CliShowIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliShowIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.cli.ShowTestCase;
+import org.elasticsearch.xpack.sql.qa.cli.ShowTestCase;
 
 public class CliShowIT extends ShowTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcConnectionIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcConnectionIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcConnectionIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcConnectionIT.java
index e75cf6d059dfe..fe34346458c8a 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcConnectionIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcConnectionIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ConnectionTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ConnectionTestCase;
 
 public class JdbcConnectionIT extends ConnectionTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
similarity index 73%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcCsvSpecIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
index a245e6c85ef24..4f841e02ae3d5 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcCsvSpecIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvSpecTestCase;
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
 public class JdbcCsvSpecIT extends CsvSpecTestCase {
     public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcDatabaseMetaDataIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDatabaseMetaDataIT.java
similarity index 73%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcDatabaseMetaDataIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDatabaseMetaDataIT.java
index 9bb663f190aff..13bc7c3695059 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcDatabaseMetaDataIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDatabaseMetaDataIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.DatabaseMetaDataTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.DatabaseMetaDataTestCase;
 
 public class JdbcDatabaseMetaDataIT extends DatabaseMetaDataTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDocCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java
similarity index 84%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDocCsvSpecIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java
index c8fc587bd6ac6..336b476a37f9b 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcDocCsvSpecIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcDocCsvSpecIT.java
@@ -3,26 +3,26 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
-import org.elasticsearch.xpack.qa.sql.jdbc.DataLoader;
-import org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert;
-import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
-import org.elasticsearch.xpack.qa.sql.jdbc.SqlSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.DataLoader;
+import org.elasticsearch.xpack.sql.qa.jdbc.JdbcAssert;
+import org.elasticsearch.xpack.sql.qa.jdbc.SpecBaseIntegrationTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.SqlSpecTestCase;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.csvConnection;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.executeCsvQuery;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.csvConnection;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
 
 /**
  * CSV test specification for DOC examples.
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcErrorsIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcErrorsIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcErrorsIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcErrorsIT.java
index 21a52b609bbb3..ba9ef20c34ad4 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcErrorsIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcErrorsIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ErrorsTestCase;
 
 public class JdbcErrorsIT extends ErrorsTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcFetchSizeIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFetchSizeIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcFetchSizeIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFetchSizeIT.java
index b64290957c088..1d08c49dae2f7 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcFetchSizeIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFetchSizeIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.FetchSizeTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.FetchSizeTestCase;
 
 public class JdbcFetchSizeIT extends FetchSizeTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcPreparedStatementIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcPreparedStatementIT.java
similarity index 73%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcPreparedStatementIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcPreparedStatementIT.java
index 0d711c2798608..9a986a33afd7f 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcPreparedStatementIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcPreparedStatementIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.PreparedStatementTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.PreparedStatementTestCase;
 
 public class JdbcPreparedStatementIT extends PreparedStatementTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcResultSetIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcResultSetIT.java
similarity index 80%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcResultSetIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcResultSetIT.java
index 30756a11f62ec..658911fb8d4d3 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcResultSetIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcResultSetIT.java
@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ResultSetTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ResultSetTestCase;
 
 /*
  * Integration testing class for "no security" (cluster running without the Security plugin,
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
similarity index 94%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
index e88b6b94d209f..487dc5f717348 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcShardFailureIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
@@ -3,10 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
 import org.elasticsearch.client.Request;
-import org.elasticsearch.xpack.qa.sql.jdbc.JdbcIntegrationTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
diff --git a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcShowTablesIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShowTablesIT.java
similarity index 72%
rename from x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcShowTablesIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShowTablesIT.java
index 7af41a5c9d8ef..f5ceefc35c1e3 100644
--- a/x-pack/qa/sql/multinode/src/test/java/org/elasticsearch/xpack/qa/sql/multinode/JdbcShowTablesIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShowTablesIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.multinode;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.ShowTablesTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.ShowTablesTestCase;
 
 public class JdbcShowTablesIT extends ShowTablesTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSimpleExampleIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSimpleExampleIT.java
similarity index 72%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSimpleExampleIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSimpleExampleIT.java
index 08539667cf92d..ca79257fc5cae 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSimpleExampleIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSimpleExampleIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.SimpleExampleTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.SimpleExampleTestCase;
 
 public class JdbcSimpleExampleIT extends SimpleExampleTestCase {
 }
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSqlSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSqlSpecIT.java
similarity index 81%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSqlSpecIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSqlSpecIT.java
index fb658270729d3..0e1867b26dfdb 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/JdbcSqlSpecIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcSqlSpecIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.SqlSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.SqlSpecTestCase;
 
 public class JdbcSqlSpecIT extends SqlSpecTestCase {
     public JdbcSqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, String query) {
diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/RestSqlIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
similarity index 79%
rename from x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/RestSqlIT.java
rename to x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
index e22c8fb085210..6bbd564e0b4cc 100644
--- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/RestSqlIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlIT.java
@@ -3,9 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.nosecurity;
+package org.elasticsearch.xpack.sql.qa.single_node;
 
-import org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase;
+import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase;
 
 /**
  * Integration test for the rest sql action. The one that speaks json directly to a
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
similarity index 96%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
index 8421ec9631150..5e3b034d75708 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql;
+package org.elasticsearch.xpack.sql.qa;
 
 /**
  * Interface implemented once per SQL access method to ensure that we
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
similarity index 93%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
index 5f992fa06e9f1..059a4ad2233b5 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/CliIntegrationTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.CheckedConsumer;
@@ -11,13 +11,13 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig;
+import org.elasticsearch.xpack.sql.qa.cli.EmbeddedCli.SecurityConfig;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.assertNoSearchContexts;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.assertNoSearchContexts;
 
 public abstract class CliIntegrationTestCase extends ESRestTestCase {
     /**
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java
similarity index 94%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java
index 234d229f324b8..251ff8665af68 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/EmbeddedCli.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -150,10 +150,21 @@ protected boolean addShutdownHook() {
                 assertEquals("", readLine());
             }
 
-            // Throw out the logo
-            while (false == readLine().contains("SQL")) {
-                ;
+            // Read until the first "good" line (skip the logo or read until an exception)
+            boolean isLogoOrException = false;
+            while (!isLogoOrException) {
+                String line = readLine();
+                if ("SQL".equals(line.trim())) {
+                    // it's almost the bottom of the logo, so read the next line (the version) and break out of the loop
+                    readLine();
+                    isLogoOrException = true;
+                } else if (line.contains("Exception")) {
+                    // if it's an exception, just break out of the loop and don't read the next line
+                    // as it will swallow the exception and IT tests won't catch it
+                    isLogoOrException = true;
+                }
             }
+            
             assertConnectionTest();
         } catch (IOException e) {
             try {
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
similarity index 95%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
index dee99a7be1ca6..03267329aa353 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.elasticsearch.client.Request;
 
@@ -14,7 +14,7 @@
 /**
  * Tests for error messages.
  */
-public abstract class ErrorsTestCase extends CliIntegrationTestCase implements org.elasticsearch.xpack.qa.sql.ErrorsTestCase {
+public abstract class ErrorsTestCase extends CliIntegrationTestCase implements org.elasticsearch.xpack.sql.qa.ErrorsTestCase {
     /**
      * Starting sequence commons to lots of errors.
      */
@@ -27,7 +27,7 @@ public abstract class ErrorsTestCase extends CliIntegrationTestCase implements o
     @Override
     public void testSelectInvalidSql() throws Exception {
         assertFoundOneProblem(command("SELECT * FRO"));
-        assertEquals("line 1:8: Cannot determine columns for *" + END, readLine());
+        assertEquals("line 1:8: Cannot determine columns for [*]" + END, readLine());
     }
 
     @Override
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
similarity index 98%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
index 19162c34e3ccd..afc4c302995d5 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/FetchSizeTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.elasticsearch.client.Request;
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/SelectTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/SelectTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/SelectTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/SelectTestCase.java
index fffa8e0f72a1d..cad607f894808 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/SelectTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/SelectTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.elasticsearch.test.hamcrest.RegexMatcher;
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java
similarity index 94%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java
index 723ca8efb94d1..00fb96d01855c 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
 
 import org.elasticsearch.test.hamcrest.RegexMatcher;
 
@@ -35,6 +35,10 @@ public void testShowFunctions() throws IOException {
         while (aggregateFunction.matcher(line).matches()) {
             line = readLine();
         }
+        Pattern conditionalFunction = Pattern.compile("\\s*[A-Z0-9_~]+\\s*\\|\\s*CONDITIONAL\\s*");
+        while (conditionalFunction.matcher(line).matches()) {
+            line = readLine();
+        }
         Pattern scalarFunction = Pattern.compile("\\s*[A-Z0-9_~]+\\s*\\|\\s*SCALAR\\s*");
         while (scalarFunction.matcher(line).matches()) {
             line = readLine();
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/package-info.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/package-info.java
similarity index 89%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/package-info.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/package-info.java
index b77e050cc3422..7eed43371d881 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/package-info.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/package-info.java
@@ -8,4 +8,4 @@
  * Support for integration tests for the Elasticsearch SQL CLI client
  * and integration tests shared between multiple qa projects. 
  */
-package org.elasticsearch.xpack.qa.sql.cli;
+package org.elasticsearch.xpack.sql.qa.cli;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ConnectionTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ConnectionTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java
index 444142b7138b0..d416d10854b6b 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ConnectionTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ConnectionTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.Version;
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java
similarity index 83%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java
index 0402b312338b9..755d701c226bf 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java
@@ -3,20 +3,20 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.csvConnection;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.executeCsvQuery;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.csvConnection;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
 
 /**
  * Tests comparing sql queries executed against our jdbc client
@@ -29,13 +29,14 @@ public abstract class CsvSpecTestCase extends SpecBaseIntegrationTestCase {
     public static List<Object[]> readScriptSpec() throws Exception {
         Parser parser = specParser();
         List<Object[]> tests = new ArrayList<>();
+        tests.addAll(readScriptSpec("/select.csv-spec", parser));
         tests.addAll(readScriptSpec("/command.csv-spec", parser));
         tests.addAll(readScriptSpec("/fulltext.csv-spec", parser));
         tests.addAll(readScriptSpec("/agg.csv-spec", parser));
         tests.addAll(readScriptSpec("/columns.csv-spec", parser));
         tests.addAll(readScriptSpec("/datetime.csv-spec", parser));
         tests.addAll(readScriptSpec("/alias.csv-spec", parser));
-        tests.addAll(readScriptSpec("/nulls.csv-spec", parser));
+        tests.addAll(readScriptSpec("/null.csv-spec", parser));
         tests.addAll(readScriptSpec("/nested.csv-spec", parser));
         tests.addAll(readScriptSpec("/functions.csv-spec", parser));
         tests.addAll(readScriptSpec("/math.csv-spec", parser));
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java
index 937a5a70b8ad0..8cc8cf6e04044 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvTestUtils.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvTestUtils.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
index 8105cbf9a5618..337706b5a5a19 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.apache.http.HttpHost;
 import org.apache.logging.log4j.LogManager;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DatabaseMetaDataTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DatabaseMetaDataTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DatabaseMetaDataTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DatabaseMetaDataTestCase.java
index a5f3a5f83643d..9e4252cc27391 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DatabaseMetaDataTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DatabaseMetaDataTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.common.CheckedSupplier;
 
@@ -12,7 +12,7 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert.assertResultSets;
+import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcAssert.assertResultSets;
 
 /**
  * Tests for our implementation of {@link DatabaseMetaData}.
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugCsvSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
similarity index 88%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugCsvSpec.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
index 4e7842dd25adf..8f07fd879b73d 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugCsvSpec.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
@@ -3,22 +3,22 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.csvConnection;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.executeCsvQuery;
-import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.csvConnection;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery;
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
 
 @TestLogging(JdbcTestUtils.SQL_TRACE)
 public abstract class DebugCsvSpec extends SpecBaseIntegrationTestCase {
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugSqlSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
similarity index 95%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugSqlSpec.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
index 8726d8ddb9cf5..21d2f3301fb96 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DebugSqlSpec.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
index 65fd0778b57f2..bb9d5f2c2fc11 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.client.Request;
 
@@ -15,12 +15,12 @@
 /**
  * Tests for exceptions and their messages.
  */
-public class ErrorsTestCase extends JdbcIntegrationTestCase implements org.elasticsearch.xpack.qa.sql.ErrorsTestCase {
+public class ErrorsTestCase extends JdbcIntegrationTestCase implements org.elasticsearch.xpack.sql.qa.ErrorsTestCase {
     @Override
     public void testSelectInvalidSql() throws Exception {
         try (Connection c = esJdbc()) {
             SQLException e = expectThrows(SQLException.class, () -> c.prepareStatement("SELECT * FRO").executeQuery());
-            assertEquals("Found 1 problem(s)\nline 1:8: Cannot determine columns for *", e.getMessage());
+            assertEquals("Found 1 problem(s)\nline 1:8: Cannot determine columns for [*]", e.getMessage());
         }
     }
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/FetchSizeTestCase.java
similarity index 96%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/FetchSizeTestCase.java
index 784fd358a4548..e0a5bd26db0a2 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/FetchSizeTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/FetchSizeTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.client.Request;
 import org.junit.Before;
@@ -14,7 +14,7 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.assertNoSearchContexts;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.assertNoSearchContexts;
 
 /**
  * Tests for setting {@link Statement#setFetchSize(int)} and
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
index 26e3ae9405e4b..3deeb1fb3c35e 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcAssert.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.apache.logging.log4j.Logger;
 import org.relique.jdbc.csv.CsvResultSet;
@@ -118,6 +118,11 @@ public static void assertResultSetMetadata(ResultSet expected, ResultSet actual,
             if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) {
                 expectedType = Types.REAL;
             }
+            // csv doesn't support NULL type so skip type checking
+            if (actualType == Types.NULL && expected instanceof CsvResultSet) {
+                expectedType = Types.NULL;
+            }
+
             // when lenient is used, an int is equivalent to a short, etc...
             assertEquals("Different column type for column [" + expectedName + "] (" + JDBCType.valueOf(expectedType) + " != "
                     + JDBCType.valueOf(actualType) + ")", expectedType, actualType);
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcIntegrationTestCase.java
similarity index 93%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcIntegrationTestCase.java
index c6594d7205112..df7e12687caa7 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcIntegrationTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
@@ -15,7 +15,6 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
 import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource;
-import org.joda.time.DateTimeZone;
 import org.junit.After;
 
 import java.io.IOException;
@@ -23,15 +22,13 @@
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
 import java.util.Set;
-import java.util.TimeZone;
 
-import static org.elasticsearch.xpack.qa.sql.rest.RestSqlTestCase.assertNoSearchContexts;
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.assertNoSearchContexts;
 
 public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
     @After
@@ -119,8 +116,8 @@ public static String randomKnownTimeZone() {
         // from all available JDK timezones. While Joda and JDK are generally in sync, some timezones might not be known
         // to the current version of Joda and in this case the test might fail. To avoid that, we specify a timezone
         // known for both Joda and JDK
-        Set<String> timeZones = new HashSet<>(DateTimeZone.getAvailableIDs());
-        timeZones.retainAll(Arrays.asList(TimeZone.getAvailableIDs()));
+        Set<String> timeZones = new HashSet<>(JODA_TIMEZONE_IDS);
+        timeZones.retainAll(JAVA_TIMEZONE_IDS);
         List<String> ids = new ArrayList<>(timeZones);
         Collections.sort(ids);
         return randomFrom(ids);
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcTestUtils.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java
index 2bb4697749a3a..7b511ee63f943 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcTestUtils.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.xpack.sql.action.CliFormatter;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/LocalH2.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/LocalH2.java
similarity index 98%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/LocalH2.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/LocalH2.java
index 8aa5a2287a805..e6295985cf519 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/LocalH2.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/LocalH2.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.CheckedSupplier;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/PreparedStatementTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/PreparedStatementTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java
index c4ac31120a3bd..34d32db16f4dc 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/PreparedStatementTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/PreparedStatementTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.common.collect.Tuple;
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ResultSetTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
index 80580f3461ac3..bd7914278b360 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ResultSetTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.CheckedBiFunction;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ShowTablesTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java
similarity index 94%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ShowTablesTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java
index aa250628f7361..ab2ddc9a7fe24 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ShowTablesTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java
@@ -3,13 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.util.Locale;
 
-import static org.elasticsearch.xpack.qa.sql.jdbc.JdbcAssert.assertResultSets;
+import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcAssert.assertResultSets;
 
 public class ShowTablesTestCase extends JdbcIntegrationTestCase {
     public void testShowTablesWithoutAnyIndexes() throws Exception {
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SimpleExampleTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SimpleExampleTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SimpleExampleTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SimpleExampleTestCase.java
index 35f2dba77792a..4d3b7698e5b5a 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SimpleExampleTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SimpleExampleTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java
index 86cbdec197e72..a7d0332508f6d 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SpecBaseIntegrationTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SpecBaseIntegrationTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Request;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
similarity index 97%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
index 10fae56be48d4..d5e720cae8614 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
@@ -41,6 +41,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
         tests.addAll(readScriptSpec("/arithmetic.sql-spec", parser));
         tests.addAll(readScriptSpec("/string-functions.sql-spec", parser));
         tests.addAll(readScriptSpec("/case-functions.sql-spec", parser));
+        tests.addAll(readScriptSpec("/null.sql-spec", parser));
         return tests;
     }
 
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/package-info.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/package-info.java
similarity index 89%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/package-info.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/package-info.java
index 1825d9033c83f..d326b0ab9daac 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/package-info.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/package-info.java
@@ -8,4 +8,4 @@
  * Support for integration tests for the Elasticsearch SQL JDBC client
  * and integration tests shared between multiple qa projects. 
  */
-package org.elasticsearch.xpack.qa.sql.jdbc;
+package org.elasticsearch.xpack.sql.qa.jdbc;
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
similarity index 99%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index a7f0bfbc0b93f..99cab175fd4e0 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.qa.sql.rest;
+package org.elasticsearch.xpack.sql.qa.rest;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
 
@@ -21,7 +21,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xpack.qa.sql.ErrorsTestCase;
+import org.elasticsearch.xpack.sql.qa.ErrorsTestCase;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
@@ -202,7 +202,7 @@ public void testSelectWhereExistsFails() throws Exception {
     @Override
     public void testSelectInvalidSql() {
         String mode = randomFrom("jdbc", "plain");
-        expectBadRequest(() -> runSql(mode, "SELECT * FRO"), containsString("1:8: Cannot determine columns for *"));
+        expectBadRequest(() -> runSql(mode, "SELECT * FRO"), containsString("1:8: Cannot determine columns for [*]"));
     }
 
     @Override
@@ -515,7 +515,8 @@ public void testTranslateQueryWithGroupByAndHaving() throws IOException {
         @SuppressWarnings("unchecked")
         Map<String, Object> filterScript = (Map<String, Object>) bucketSelector.get("script");
         assertEquals(3, filterScript.size());
-        assertEquals("InternalSqlScriptUtils.gt(params.a0,params.v0)", filterScript.get("source"));
+        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.gt(params.a0,params.v0))",
+            filterScript.get("source"));
         assertEquals("painless", filterScript.get("lang"));
         @SuppressWarnings("unchecked")
         Map<String, Object> filterScriptParams = (Map<String, Object>) filterScript.get("params");
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/package-info.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/package-info.java
similarity index 87%
rename from x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/package-info.java
rename to x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/package-info.java
index 1a061730c60bf..a07a23c7b0488 100644
--- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/package-info.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/package-info.java
@@ -7,4 +7,4 @@
 /**
  * Integration tests shared between multiple qa projects. 
  */
-package org.elasticsearch.xpack.qa.sql.rest;
+package org.elasticsearch.xpack.sql.qa.rest;
diff --git a/x-pack/qa/sql/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/agg.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/agg.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec
similarity index 99%
rename from x-pack/qa/sql/src/main/resources/agg.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec
index 9e4748d266f82..9adbe79edc685 100644
--- a/x-pack/qa/sql/src/main/resources/agg.sql-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec
@@ -129,6 +129,10 @@ aggCountAndHaving
 SELECT gender g, COUNT(*) c FROM "test_emp" GROUP BY g HAVING COUNT(*) > 10 ORDER BY gender;
 aggCountAndHavingEquality
 SELECT gender g, COUNT(*) c FROM "test_emp" GROUP BY g HAVING COUNT(*) = 10 ORDER BY gender;
+aggCountAndHavingNotEquals
+SELECT gender g, COUNT(*) c FROM "test_emp" GROUP BY g HAVING COUNT(*) != 10 ORDER BY gender;
+aggCountAndHavingNegateEquality
+SELECT gender g, COUNT(*) c FROM "test_emp" GROUP BY g HAVING NOT COUNT(*) = 10 ORDER BY gender;
 aggCountOnColumnAndHaving
 SELECT gender g, COUNT(gender) c FROM "test_emp" GROUP BY g HAVING COUNT(gender) > 10 ORDER BY gender;
 aggCountOnColumnAndWildcardAndHaving
diff --git a/x-pack/qa/sql/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/alias.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/arithmetic.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/arithmetic.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/arithmetic.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/arithmetic.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/arithmetic.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/case-functions.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/case-functions.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/case-functions.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/case-functions.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/columns.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/columns.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/columns.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/columns.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command-sys.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/command-sys.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/command-sys.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
similarity index 99%
rename from x-pack/qa/sql/src/main/resources/command.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
index 0514b2a4982bb..d15e849921592 100644
--- a/x-pack/qa/sql/src/main/resources/command.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
@@ -19,6 +19,7 @@ SKEWNESS        |AGGREGATE
 STDDEV_POP      |AGGREGATE      
 SUM_OF_SQUARES  |AGGREGATE      
 VAR_POP         |AGGREGATE      
+COALESCE        |CONDITIONAL
 DAY             |SCALAR         
 DAYNAME         |SCALAR         
 DAYOFMONTH      |SCALAR         
diff --git a/x-pack/qa/sql/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/datetime.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/datetime.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/debug.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/debug.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/debug.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/debug.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/debug.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/debug.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/debug.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/debug.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/dep_emp.csv b/x-pack/plugin/sql/qa/src/main/resources/dep_emp.csv
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/dep_emp.csv
rename to x-pack/plugin/sql/qa/src/main/resources/dep_emp.csv
diff --git a/x-pack/qa/sql/src/main/resources/departments.csv b/x-pack/plugin/sql/qa/src/main/resources/departments.csv
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/departments.csv
rename to x-pack/plugin/sql/qa/src/main/resources/departments.csv
diff --git a/x-pack/qa/sql/src/main/resources/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec
similarity index 99%
rename from x-pack/qa/sql/src/main/resources/docs.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec
index bba77664ceda8..ccb9498f3590a 100644
--- a/x-pack/qa/sql/src/main/resources/docs.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec
@@ -196,6 +196,7 @@ SKEWNESS        |AGGREGATE
 STDDEV_POP      |AGGREGATE      
 SUM_OF_SQUARES  |AGGREGATE      
 VAR_POP         |AGGREGATE      
+COALESCE        |CONDITIONAL
 DAY             |SCALAR         
 DAYNAME         |SCALAR         
 DAYOFMONTH      |SCALAR         
diff --git a/x-pack/qa/sql/src/main/resources/employees.csv b/x-pack/plugin/sql/qa/src/main/resources/employees.csv
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/employees.csv
rename to x-pack/plugin/sql/qa/src/main/resources/employees.csv
diff --git a/x-pack/qa/sql/src/main/resources/example.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/example.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/example.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/example.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/example.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/example.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/example.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/example.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/filter.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/fulltext.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/fulltext.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/functions.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/functions.csv-spec
similarity index 96%
rename from x-pack/qa/sql/src/main/resources/functions.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/functions.csv-spec
index 6e6ccf872b585..930a15f9438a4 100644
--- a/x-pack/qa/sql/src/main/resources/functions.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/functions.csv-spec
@@ -232,21 +232,13 @@ SELECT POSITION('x',LCASE("first_name")) pos, "first_name" FROM "test_emp" WHERE
       pos:i    |  first_name:s
 ---------------+---------------
 4              |Guoxiang       
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-null           |null           
-1              |Xinglin  
+1              |Xinglin
 ;
 
 selectPositionWithLcaseAndConditionWithGroupByAndOrderBy
-SELECT POSITION('m',LCASE("first_name")), COUNT(*) pos FROM "test_emp" WHERE POSITION('m',LCASE("first_name")) != 0 GROUP BY POSITION('m',LCASE("first_name")) ORDER BY POSITION('m',LCASE("first_name")) DESC;
+SELECT POSITION('m',LCASE("first_name")), COUNT(*) pos FROM "test_emp"
+  WHERE POSITION('m',LCASE("first_name")) != 0
+  GROUP BY POSITION('m',LCASE("first_name")) ORDER BY POSITION('m',LCASE("first_name")) DESC;
 
 POSITION(m,LCASE(first_name)):i|      pos:l
 -------------------------------+---------------
@@ -256,7 +248,6 @@ POSITION(m,LCASE(first_name)):i|      pos:l
 3                              |6              
 2                              |1              
 1                              |9              
-null                           |10    
 ;
 
 selectInsertWithPositionAndCondition
diff --git a/x-pack/qa/sql/src/main/resources/ip.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/ip.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/ip.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/ip.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/library.csv b/x-pack/plugin/sql/qa/src/main/resources/library.csv
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/library.csv
rename to x-pack/plugin/sql/qa/src/main/resources/library.csv
diff --git a/x-pack/qa/sql/src/main/resources/logs.csv b/x-pack/plugin/sql/qa/src/main/resources/logs.csv
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/logs.csv
rename to x-pack/plugin/sql/qa/src/main/resources/logs.csv
diff --git a/x-pack/qa/sql/src/main/resources/math.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/math.csv-spec
rename to x-pack/plugin/sql/qa/src/main/resources/math.csv-spec
diff --git a/x-pack/qa/sql/src/main/resources/math.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/math.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/math.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/math.sql-spec
diff --git a/x-pack/plugin/sql/qa/src/main/resources/nested.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/nested.csv-spec
new file mode 100644
index 0000000000000..89808901e9cff
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/nested.csv-spec
@@ -0,0 +1,224 @@
+//
+//  Nested documents
+//
+// CsvJdbc has issues with foo.bar so most fields are aliases or wrapped inside a function
+
+describeParent
+DESCRIBE test_emp;
+
+       column       |     type      |    mapping    
+--------------------+---------------+---------------
+birth_date          |TIMESTAMP      |DATE           
+dep                 |STRUCT         |NESTED         
+dep.dep_id          |VARCHAR        |KEYWORD        
+dep.dep_name        |VARCHAR        |TEXT           
+dep.dep_name.keyword|VARCHAR        |KEYWORD        
+dep.from_date       |TIMESTAMP      |DATE           
+dep.to_date         |TIMESTAMP      |DATE           
+emp_no              |INTEGER        |INTEGER        
+first_name          |VARCHAR        |TEXT           
+first_name.keyword  |VARCHAR        |KEYWORD        
+gender              |VARCHAR        |KEYWORD        
+hire_date           |TIMESTAMP      |DATE           
+languages           |TINYINT        |BYTE           
+last_name           |VARCHAR        |TEXT           
+last_name.keyword   |VARCHAR        |KEYWORD        
+salary              |INTEGER        |INTEGER   
+;
+
+nestedStar
+SELECT dep.* FROM test_emp ORDER BY dep.dep_id LIMIT 5;
+
+  dep.dep_id:s   | dep.dep_name:s |    dep.from_date:ts    |     dep.to_date:ts
+
+d001             |Marketing       |1993-08-03T00:00:00.000Z|9999-01-01T00:00:00.000Z
+d001             |Marketing       |1992-04-27T00:00:00.000Z|1995-07-22T00:00:00.000Z
+d001             |Marketing       |1988-04-25T00:00:00.000Z|9999-01-01T00:00:00.000Z
+d002             |Finance         |1993-03-21T00:00:00.000Z|2000-08-10T00:00:00.000Z
+d002             |Finance         |1990-12-25T00:00:00.000Z|1992-11-05T00:00:00.000Z
+;
+
+// Test for https://github.com/elastic/elasticsearch/issues/30054
+nestedSingleFieldSelect
+SELECT dep.dep_id FROM test_emp LIMIT 5;
+
+ dep.dep_id:s   
+
+d005           
+d007           
+d004           
+d004           
+d003           
+;
+
+filterPerNestedWithOrderByTopLevel
+SELECT first_name f, last_name l, YEAR(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY f LIMIT 5;
+
+f:s                  | l:s                  | d:i
+
+Chirstian            |Koblick               |1986           
+Duangkaew            |Piveteau              |1996           
+Gino                 |Leonhardt             |1989           
+Hidefumi             |Caine                 |1992           
+Jayson               |Mandell               |1999    
+;
+
+filterPerNestedWithOrderByNested
+SELECT first_name f, last_name l, YEAR(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY dep.from_date LIMIT 5;
+
+f:s                  | l:s                  | d:i
+
+Sreekrishna          |Servieres             |1985           
+Zhongwei             |Rosen                 |1986           
+Chirstian            |Koblick               |1986           
+Vishv                |Zockler               |1987           
+null                 |Chappelet             |1988    
+;
+
+filterPerNestedWithOrderByNestedWithAlias
+SELECT first_name f, dep.dep_id i, MONTH(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY i LIMIT 5;
+
+f:s                  | i:s                  | d:i
+
+Parto                | d004                 | 12
+Chirstian            | d004                 | 12
+Duangkaew            | d004                 | 11
+Kazuhide             | d004                 | 7
+Mayuko               | d004                 | 12
+;
+
+filterPerNestedWithOrderByNestedWithoutProjection
+SELECT first_name f, MONTH(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY dep.dep_id LIMIT 5;
+
+f:s        | d:i
+
+Parto      | 12
+Chirstian  | 12
+Duangkaew  | 11
+Kazuhide   | 7
+Mayuko     | 12
+;
+
+selectWithScalarOnNested
+SELECT first_name f, last_name l, YEAR(dep.from_date) start FROM test_emp WHERE dep.dep_name = 'Production' AND languages > 1 ORDER BY start LIMIT 5;
+
+f:s                  | l:s                  | start:i
+
+Sreekrishna          |Servieres      |1985           
+Zhongwei             |Rosen          |1986           
+Chirstian            |Koblick        |1986           
+null                 |Chappelet      |1988           
+Zvonko               |Nyanchama      |1989     
+;
+
+selectWithScalarOnNestedWithoutProjection
+SELECT first_name f, last_name l FROM test_emp WHERE dep.dep_name = 'Production' AND languages > 1 ORDER BY YEAR(dep.from_date) LIMIT 5;
+
+f:s                  | l:s
+
+Sreekrishna          |Servieres      
+Zhongwei             |Rosen          
+Chirstian            |Koblick        
+null                 |Chappelet      
+Zvonko               |Nyanchama  
+;
+
+//
+// Tests for bug https://github.com/elastic/elasticsearch/issues/32951 fix
+//
+
+selectNestedFieldFirst
+SELECT dep.dep_id, last_name FROM test_emp ORDER BY last_name LIMIT 5;
+
+dep.dep_id:s   |   last_name:s   
+
+d005           |Awdeh
+d003           |Azuma
+d002           |Baek
+d003           |Baek
+d004           |Bamford
+; 
+
+selectNestedFieldLast
+SELECT first_name, dep.dep_id FROM test_emp ORDER BY first_name LIMIT 5;
+
+first_name:s   |  dep.dep_id:s
+---------------+---------------
+Alejandro      |d002           
+Amabile        |d005           
+Anneke         |d005           
+Anoosh         |d005           
+Arumugam       |d008           
+;
+
+selectNestedFieldInTheMiddle
+SELECT first_name, dep.dep_name, last_name FROM test_emp ORDER BY first_name LIMIT 5;
+
+first_name:s   |dep.dep_name:s |last_name:s
+
+Alejandro      |Finance        |McAlpine
+Amabile        |Development    |Gomatam
+Anneke         |Development    |Preusig
+Anoosh         |Development    |Peyn
+Arumugam       |Research       |Ossenbruggen
+;
+
+selectNestedFieldInTheMiddleAndAtTheEnd
+SELECT first_name, dep.dep_name, last_name, dep.dep_id FROM test_emp ORDER BY first_name LIMIT 5;
+
+ first_name:s  |dep.dep_name:s | last_name:s   |  dep.dep_id:s
+
+Alejandro      |Finance        |McAlpine       |d002
+Amabile        |Development    |Gomatam        |d005
+Anneke         |Development    |Preusig        |d005
+Anoosh         |Development    |Peyn           |d005
+Arumugam       |Research       |Ossenbruggen   |d008
+;
+
+selectNestedFieldInTheMiddleAndAtBeggining
+SELECT dep.dep_id, first_name, dep.dep_name, last_name FROM test_emp ORDER BY first_name LIMIT 5;
+
+ dep.dep_id:s  | first_name:s  |dep.dep_name:s | last_name:s
+
+d002           |Alejandro      |Finance        |McAlpine
+d005           |Amabile        |Development    |Gomatam
+d005           |Anneke         |Development    |Preusig
+d005           |Anoosh         |Development    |Peyn
+d008           |Arumugam       |Research       |Ossenbruggen
+;
+
+selectNestedFieldWithWildcardAtBeggining
+SELECT dep.*, first_name FROM test_emp ORDER BY first_name LIMIT 5;
+
+ dep.dep_id:s  |dep.dep_name:s |   dep.from_date:ts     |    dep.to_date:ts      |  first_name:s
+
+d002           |Finance        |1991-06-26T00:00:00.000Z|9999-01-01T00:00:00.000Z|Alejandro
+d005           |Development    |1992-11-18T00:00:00.000Z|9999-01-01T00:00:00.000Z|Amabile
+d005           |Development    |1990-08-05T00:00:00.000Z|9999-01-01T00:00:00.000Z|Anneke
+d005           |Development    |1991-08-30T00:00:00.000Z|9999-01-01T00:00:00.000Z|Anoosh
+d008           |Research       |1987-04-18T00:00:00.000Z|1997-11-08T00:00:00.000Z|Arumugam
+;
+
+selectNestedFieldWithWildcardAtTheEnd
+SELECT first_name, dep.* FROM test_emp ORDER BY first_name LIMIT 5;
+
+ first_name:s  | dep.dep_id:s  |dep.dep_name:s |   dep.from_date:ts     |     dep.to_date:ts
+
+Alejandro      |d002           |Finance        |1991-06-26T00:00:00.000Z|9999-01-01T00:00:00.000Z
+Amabile        |d005           |Development    |1992-11-18T00:00:00.000Z|9999-01-01T00:00:00.000Z
+Anneke         |d005           |Development    |1990-08-05T00:00:00.000Z|9999-01-01T00:00:00.000Z
+Anoosh         |d005           |Development    |1991-08-30T00:00:00.000Z|9999-01-01T00:00:00.000Z
+Arumugam       |d008           |Research       |1987-04-18T00:00:00.000Z|1997-11-08T00:00:00.000Z
+;
+
+selectNestedFieldWithWildcardInTheMiddle
+SELECT first_name, dep.*, last_name FROM test_emp ORDER BY first_name LIMIT 5;
+
+ first_name:s  | dep.dep_id:s  |dep.dep_name:s |   dep.from_date:ts     |    dep.to_date:ts      |   last_name:s
+
+Alejandro      |d002           |Finance        |1991-06-26T00:00:00.000Z|9999-01-01T00:00:00.000Z|McAlpine
+Amabile        |d005           |Development    |1992-11-18T00:00:00.000Z|9999-01-01T00:00:00.000Z|Gomatam
+Anneke         |d005           |Development    |1990-08-05T00:00:00.000Z|9999-01-01T00:00:00.000Z|Preusig
+Anoosh         |d005           |Development    |1991-08-30T00:00:00.000Z|9999-01-01T00:00:00.000Z|Peyn
+Arumugam       |d008           |Research       |1987-04-18T00:00:00.000Z|1997-11-08T00:00:00.000Z|Ossenbruggen
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/null.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/null.csv-spec
new file mode 100644
index 0000000000000..474fceaed4612
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/null.csv-spec
@@ -0,0 +1,75 @@
+//
+// Null expressions
+//
+
+dateTimeOverNull
+SELECT YEAR(CAST(NULL AS DATE)) d;
+
+d:i
+null
+;
+
+addOfNull
+SELECT CAST(NULL AS INT) + CAST(NULL AS FLOAT) AS n;
+
+n:d
+null
+;
+
+
+divOfCastedNull
+SELECT 5 / CAST(NULL AS FLOAT) + 10 AS n;
+
+n:d
+null
+;
+
+divNoNull
+SELECT 5 / null + 1 AS n;
+
+n:i
+null
+;
+
+coalesceJustWithNull
+SELECT COALESCE(null, null, null) AS c;
+
+c
+null
+;
+
+coalesceFirstNotNull
+SELECT COALESCE(123) AS c;
+
+c
+123
+;
+
+
+coalesceWithFirstNullOfString
+SELECT COALESCE(null, 'first') AS c;
+
+c:s
+first
+;
+
+coalesceWithFirstNullOfNumber
+SELECT COALESCE(null, 123) AS c;
+
+c:i
+123
+;
+
+coalesceMixed
+SELECT COALESCE(null, 123, null, 321) AS c;
+
+c:i
+123
+;
+
+coalesceScalar
+SELECT COALESCE(null, ABS(123) + 1) AS c;
+
+c:i
+124
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/null.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/null.sql-spec
new file mode 100644
index 0000000000000..8976c8d8a5ea9
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/null.sql-spec
@@ -0,0 +1,12 @@
+//
+// Null expressions
+//
+
+coalesceField
+SELECT COALESCE(null, ABS(emp_no) + 1) AS c FROM test_emp ORDER BY emp_no LIMIT 5;
+
+coalesceHaving
+SELECT COALESCE(null, ABS(MAX(emp_no)) + 1, 123) AS c FROM test_emp GROUP BY languages HAVING c > 100 ORDER BY languages LIMIT 5;
+
+coalesceWhere
+SELECT COALESCE(null, ABS(emp_no) + 1, 123) AS c FROM test_emp WHERE COALESCE(null, ABS(emp_no) + 1, 123, 321) > 100 ORDER BY emp_no NULLS FIRST LIMIT 5;
diff --git a/x-pack/qa/sql/src/main/resources/plugin-security.policy b/x-pack/plugin/sql/qa/src/main/resources/plugin-security.policy
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/plugin-security.policy
rename to x-pack/plugin/sql/qa/src/main/resources/plugin-security.policy
diff --git a/x-pack/plugin/sql/qa/src/main/resources/select.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/select.csv-spec
new file mode 100644
index 0000000000000..2aa7d9bdc7b51
--- /dev/null
+++ b/x-pack/plugin/sql/qa/src/main/resources/select.csv-spec
@@ -0,0 +1,187 @@
+//
+// SELECT with = and !=
+//
+// Need to CAST as STRING since for boolean types jdbc CSV translates null -> false
+equalsSelectClause
+SELECT CAST(4 = 4 AS STRING), CAST(NOT 4 = 4 AS STRING), CAST(3 = 4 AS STRING), CAST(NOT 3 = 4 AS STRING), CAST(1 = null AS STRING), CAST(NOT null = 1 AS STRING);
+
+  CAST(4 == 4 AS VARCHAR):s |  CAST(NOT(4 == 4) AS VARCHAR):s |  CAST(3 == 4 AS VARCHAR):s |  CAST(NOT(3 == 4) AS VARCHAR):s |  CAST(1 == null AS VARCHAR):s |  CAST(NOT(null == 1) AS VARCHAR):s
+----------------------------+---------------------------------+----------------------------+---------------------------------+-------------------------------+-----------------------------------
+true                        |false                            |false                       |true                             |null                           |null
+;
+
+notEqualsSelectClause
+SELECT CAST(4 != 4 AS STRING), CAST(NOT 4 != 4 AS STRING), CAST(3 != 4 AS STRING), CAST(NOT 3 != 4 AS STRING), CAST(1 != null AS STRING), CAST(NOT 1 != null AS STRING);
+
+  CAST(4 != 4 AS VARCHAR):s |  CAST(NOT(4 != 4) AS VARCHAR):s |  CAST(3 != 4 AS VARCHAR):s |  CAST(NOT(3 != 4) AS VARCHAR):s |  CAST(1 != null AS VARCHAR):s |  CAST(NOT(1 != null) AS VARCHAR):s
+----------------------------+---------------------------------+----------------------------+---------------------------------+-------------------------------+-----------------------------------
+false                       |true                             |true                        |false                            |null                           |null
+;
+
+equalSelectClauseWithTableColumns
+SELECT CAST(languages = 2 AS STRING), CAST(NOT languages = 2 AS STRING), CAST(languages = null AS STRING), CAST(NOT languages = null AS STRING)
+FROM "test_emp" WHERE emp_no IN(10018, 10019, 10020) ORDER BY emp_no;
+
+  CAST((languages) == 2 AS VARCHAR):s |  CAST(NOT((languages) == 2) AS VARCHAR):s |  CAST((languages) == null AS VARCHAR):s |  CAST(NOT((languages) == null) AS VARCHAR):s
+--------------------------------------+-------------------------------------------+-----------------------------------------+---------------------------------------------
+true                                  |false                                      |null                                     |null
+false                                 |true                                       |null                                     |null
+null                                  |null                                       |null                                     |null
+;
+
+notEqualsAndNotEqualsSelectClauseWithTableColumns
+SELECT CAST(languages != 2 AS STRING), CAST(NOT languages != 2 AS STRING), CAST(languages != null AS STRING), CAST(NOT languages != null AS STRING)
+FROM "test_emp" WHERE emp_no IN(10018, 10019, 10020) ORDER BY emp_no;
+
+  CAST((languages) != 2 AS VARCHAR):s |  CAST(NOT((languages) != 2) AS VARCHAR):s |  CAST((languages) != null AS VARCHAR):s |  CAST(NOT((languages) != null) AS VARCHAR):s
+--------------------------------------+-------------------------------------------+-----------------------------------------+---------------------------------------------
+false                                 |true                                       |null                                     |null
+true                                  |false                                      |null                                     |null
+null                                  |null                                       |null                                     |null
+;
+
+
+//
+// SELECT with OR and AND and NULL handling
+//
+// Need to CAST as STRING since for boolean types jdbc CSV translates null -> false
+selectWithOrAndNullHandling
+SELECT CAST(true OR null AS STRING), CAST(null OR true AS STRING), CAST(false OR null AS STRING), CAST(null OR false AS STRING), CAST(null OR null AS STRING);
+
+  CAST(true OR null AS VARCHAR):s |  CAST(null OR true AS VARCHAR):s |  CAST(false OR null AS VARCHAR):s |  CAST(null OR false AS VARCHAR):s |  CAST(null OR null AS VARCHAR):s
+----------------------------------+----------------------------------+-----------------------------------+-----------------------------------+---------------------------------
+true                              |true                              |null                               |null                               |null
+;
+
+selectWithAndAndNullHandling
+SELECT CAST(true AND null AS STRING), CAST(null AND true AS STRING), CAST(false AND null AS STRING), CAST(null AND false AS STRING), CAST(null AND null AS STRING);
+
+  CAST(true AND null AS VARCHAR):s |  CAST(null AND true AS VARCHAR):s |  CAST(false AND null AS VARCHAR):s |  CAST(null AND false AS VARCHAR):s |  CAST(null AND null AS VARCHAR):s
+-----------------------------------+-----------------------------------+------------------------------------+------------------------------------+----------------------------------
+null                               |null                               |false                               |false                               |null
+;
+
+selectWithOrAndAndAndNullHandling_WithTableColumns
+SELECT CAST(languages = 2 OR null AS STRING), CAST(languages = 2 AND null AS STRING) FROM test_emp WHERE emp_no BETWEEN 10018 AND 10020 ORDER BY emp_no;
+
+ CAST(((languages) == 2) OR null AS VARCHAR):s |  CAST(((languages) == 2) AND null AS VARCHAR):s
+-----------------------------------------------+------------------------------------------------
+true                                           |null
+null                                           |false
+null                                           |null
+;
+
+
+//
+// SELECT with IN
+//
+inWithLiterals
+SELECT 1 IN (1, 2, 3), 1 IN (2, 3);
+
+  1 IN (1, 2, 3):b |  1 IN (2, 3):b
+-------------------+-------------
+true               |false
+;
+
+inWithLiteralsAndFunctions
+SELECT 1 IN (2 - 1, 2, 3), abs(-1) IN (2, 3, abs(4 - 5));
+
+  1 IN (2 - 1, 2, 3) |  ABS(-1) IN (2, 3, ABS(4 - 5))
+---------------------+------------------------------
+true                 |true
+;
+
+
+inWithLiteralsAndNegation
+SELECT 1 NOT IN (1, 1 + 1, 3), 1 NOT IN (2, 3);
+
+  NOT(1 IN (1, 1 + 1, 3)) | NOT(1 IN (2, 3))
+--------------------------+-----------------
+false                     |true
+;
+
+// Need to CAST as STRING since for boolean types the jdbc CSV translates null -> false
+inWithNullHandling
+SELECT CAST(2 IN (1, null, 3) AS STRING), CAST(3 IN (1, null, 3) AS STRING), CAST(null IN (1, null, 3) AS STRING), CAST(null IN (1, 2, 3) AS STRING);
+
+  CAST(2 IN (1, null, 3) AS VARCHAR):s | CAST(3 IN (1, null, 3) AS VARCHAR):s |  CAST(null IN (1, null, 3) AS VARCHAR):s |  CAST(null IN (1, 2, 3) AS VARCHAR):s
+---------------------------------------+--------------------------------------+------------------------------------------+--------------------------------------
+null                                   |true                                  |null                                      |null
+;
+
+inWithNullHandlingAndNegation
+SELECT CAST(NOT 2 IN (1, null, 3) AS STRING), CAST(3 NOT IN (1, null, 3) AS STRING), CAST(NOT null IN (1, null, 3) AS STRING), CAST(null NOT IN (1, 2, 3) AS STRING);
+
+  CAST(NOT(2 IN (1, null, 3)) AS VARCHAR):s |  CAST(NOT(3 IN (1, null, 3)) AS VARCHAR):s |  CAST(NOT(null IN (1, null, 3)) AS VARCHAR):s |  CAST(NOT(null IN (1, 2, 3)) AS VARCHAR):s
+--------------------------------------------+--------------------------------------------+-----------------------------------------------+-------------------------------------------
+null                                        |false                                       |null                                           |null
+;
+
+//
+// SELECT with IN and table columns
+//
+inWithTableColumn
+SELECT emp_no IN (10000, 10001, 10002) FROM test_emp WHERE emp_no BETWEEN 10001 AND 10004 ORDER BY emp_no;
+
+ emp_no IN (10000, 10001, 10002):b
+----------------------------------
+true
+true
+false
+false
+;
+
+inWithTableColumnAndFunction
+SELECT emp_no IN (10000, 10000 + 1, abs(-10000 - 2)) FROM test_emp WHERE emp_no BETWEEN 10001 AND 10004 ORDER BY emp_no;
+
+ emp_no IN (10000, 10000 + 1, ABS(-10000 - 2)):b
+------------------------------------------------
+true
+true
+false
+false
+;
+
+inWithTableColumnAndNegation
+SELECT emp_no NOT IN (10000, 10000 + 1, 10002) FROM test_emp WHERE emp_no BETWEEN 10001 AND 10004 ORDER BY emp_no;
+
+ NOT(emp_no IN (10000, 10000 + 1, 10002)):b
+-------------------------------------------
+false
+false
+true
+true
+;
+
+inWithTableColumnAndComplexFunctions
+SELECT emp_no IN (1, abs(1 - 10002), 3) OR emp_no NOT IN (10000, 10000 + 2, 10003) FROM test_emp WHERE emp_no BETWEEN 10001 AND 10004 ORDER BY emp_no;
+
+(emp_no IN (1, ABS(1 - 10002), 3)) OR (NOT(emp_no IN (10000, 10000 + 2, 10003))):b
+----------------------------------------------------------------------------------
+true
+false
+false
+true
+;
+
+
+// Need to CAST as STRING since for boolean types the jdbc CSV translates null -> false
+inWithTableColumnAndNullHandling
+SELECT emp_no, CAST(languages IN (2, 3) AS STRING), CAST(languages IN (2, null, 3) AS STRING) FROM test_emp WHERE emp_no BETWEEN 10018 AND 10020 ORDER BY emp_no;
+
+ emp_no:i |  CAST(languages IN (2, 3) AS VARCHAR):s |  CAST(languages IN (2, null, 3) AS VARCHAR):s
+----------+-----------------------------------------+----------------------------------------------
+10018     |true                                     |true
+10019     |false                                    |null
+10020     |null                                     |null
+;
+
+inWithTableColumnAndNullHandlingAndNegation
+SELECT emp_no, CAST(languages NOT IN (2, 3) AS STRING), CAST(NOT languages IN (2, null, 3) AS STRING) FROM test_emp WHERE emp_no BETWEEN 10018 AND 10020 ORDER BY emp_no;
+
+ emp_no:i |  CAST(NOT(languages IN (2, 3)) AS VARCHAR):s |  CAST(NOT(languages IN (2, null, 3)) AS VARCHAR):s
+----------+----------------------------------------------+---------------------------------------------------
+10018     |false                                         |false
+10019     |true                                          |null
+10020     |null                                          |null
+;
diff --git a/x-pack/qa/sql/src/main/resources/select.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/select.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/select.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/select.sql-spec
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_columns.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_columns.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_procedure_columns.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_procedure_columns.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_procedure_columns.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_procedure_columns.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_procedures.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_procedures.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_procedures.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_procedures.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_table_types.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_table_types.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_table_types.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_table_types.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_tables.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_tables.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_metadata_get_tables.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_metadata_get_tables.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_mock_show_tables.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_mock_show_tables.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_mock_show_tables.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_mock_show_tables.sql
diff --git a/x-pack/qa/sql/src/main/resources/setup_test_emp.sql b/x-pack/plugin/sql/qa/src/main/resources/setup_test_emp.sql
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/setup_test_emp.sql
rename to x-pack/plugin/sql/qa/src/main/resources/setup_test_emp.sql
diff --git a/x-pack/qa/sql/src/main/resources/string-functions.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/string-functions.sql-spec
similarity index 100%
rename from x-pack/qa/sql/src/main/resources/string-functions.sql-spec
rename to x-pack/plugin/sql/qa/src/main/resources/string-functions.sql-spec
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0.jar.sha1
deleted file mode 100644
index d3e7ba985dfed..0000000000000
--- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-736e94305e2a0e803563c5b184877df5c7d4cb69
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1
new file mode 100644
index 0000000000000..d8d15caa0b4c3
--- /dev/null
+++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-7.6.0-snapshot-f9598f335b.jar.sha1
@@ -0,0 +1 @@
+ec080318251807172626c0ba8dc931276d52b15e
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle
index f0022040b4909..49f2dcef3d95c 100644
--- a/x-pack/plugin/sql/sql-cli/build.gradle
+++ b/x-pack/plugin/sql/sql-cli/build.gradle
@@ -16,7 +16,7 @@ archivesBaseName = 'elasticsearch-sql-cli'
 description = 'Command line interface to Elasticsearch that speaks SQL'
 
 dependencies {
-    compile "org.jline:jline:3.6.0"
+    compile "org.jline:jline:3.8.2"
     compile xpackProject('plugin:sql:sql-client')
     compile xpackProject('plugin:sql:sql-action')
     compile "org.elasticsearch:elasticsearch-cli:${version}"
diff --git a/x-pack/plugin/sql/sql-cli/licenses/jline-3.6.0.jar.sha1 b/x-pack/plugin/sql/sql-cli/licenses/jline-3.6.0.jar.sha1
deleted file mode 100644
index d6938e6d80b0d..0000000000000
--- a/x-pack/plugin/sql/sql-cli/licenses/jline-3.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c8ecc302d6b7d19da41c66be7d428c17cd6b12b2
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-cli/licenses/jline-3.8.2.jar.sha1 b/x-pack/plugin/sql/sql-cli/licenses/jline-3.8.2.jar.sha1
new file mode 100644
index 0000000000000..29e11fa3a021e
--- /dev/null
+++ b/x-pack/plugin/sql/sql-cli/licenses/jline-3.8.2.jar.sha1
@@ -0,0 +1 @@
+8b81efadcb78388b662ede7965b272be56a86ec1
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
index 306189b535a03..40cce57321d65 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
+++ b/x-pack/plugin/sql/sql-cli/src/main/java/org/elasticsearch/xpack/sql/cli/command/PrintLogoCommand.java
@@ -8,12 +8,14 @@
 import org.elasticsearch.xpack.sql.cli.Cli;
 import org.elasticsearch.xpack.sql.cli.CliTerminal;
 import org.elasticsearch.xpack.sql.cli.FatalCliException;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -34,6 +36,7 @@ protected boolean doHandle(CliTerminal terminal, CliSession cliSession, Matcher
 
     public void printLogo(CliTerminal terminal) {
         terminal.clear();
+        int lineLength = 0;
         try (InputStream in = Cli.class.getResourceAsStream("/logo.txt")) {
             if (in == null) {
                 throw new FatalCliException("Could not find logo!");
@@ -41,6 +44,7 @@ public void printLogo(CliTerminal terminal) {
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
                 String line;
                 while ((line = reader.readLine()) != null) {
+                    lineLength = Math.max(lineLength, line.length());
                     terminal.println(line);
                 }
             }
@@ -48,6 +52,10 @@ public void printLogo(CliTerminal terminal) {
             throw new FatalCliException("Could not load logo!", e);
         }
 
+        // print the version centered on the last line
+        char[] whitespaces = new char[(lineLength - Version.CURRENT.version.length()) / 2];
+        Arrays.fill(whitespaces, ' ');
+        terminal.println(new StringBuilder().append(whitespaces).append(Version.CURRENT.version).toString());
         terminal.println();
     }
 
diff --git a/x-pack/plugin/sql/sql-cli/src/main/resources/logo.txt b/x-pack/plugin/sql/sql-cli/src/main/resources/logo.txt
index 2880a1d41534d..bcf746c40c63e 100644
--- a/x-pack/plugin/sql/sql-cli/src/main/resources/logo.txt
+++ b/x-pack/plugin/sql/sql-cli/src/main/resources/logo.txt
@@ -1 +1,27 @@
-                 Elasticsearch SQL
\ No newline at end of file
+                       asticElasticE              
+                     ElasticE  sticEla            
+          sticEl  ticEl            Elast          
+        lasti Elasti                   tic        
+      cEl       ast                     icE       
+     icE        as                       cEl      
+     icE        as                       cEl      
+     icEla     las                        El      
+   sticElasticElast                     icElas    
+ las           last                    ticElast   
+El              asti                 asti    stic 
+El              asticEla           Elas        icE
+El            Elas  cElasticE   ticEl           cE
+Ela        ticEl         ticElasti              cE
+ las     astic               last              icE
+   sticElas                   asti           stic 
+     icEl                      sticElasticElast   
+     icE                       sticE   ticEla     
+     icE                       sti       cEla     
+     icEl                      sti        Ela     
+      cEl                      sti       cEl      
+       Ela                    astic    ticE       
+         asti               ElasticElasti         
+           ticElasti  lasticElas                  
+              ElasticElast                        
+
+                       SQL
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
index b6a54a3a8cf5d..39c48bc31d926 100644
--- a/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
+++ b/x-pack/plugin/sql/sql-cli/src/test/java/org/elasticsearch/xpack/sql/cli/command/BuiltinCommandTests.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.cli.TestTerminal;
 import org.elasticsearch.xpack.sql.client.HttpClient;
+import org.elasticsearch.xpack.sql.client.Version;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Mockito.mock;
@@ -93,6 +94,7 @@ public void testPrintLogo() throws Exception {
         testTerminal.print("not clean");
         assertTrue(new PrintLogoCommand().handle(testTerminal, cliSession, "logo"));
         assertThat(testTerminal.toString(), containsString("SQL"));
+        assertThat(testTerminal.toString(), containsString(Version.CURRENT.version));
         verifyNoMoreInteractions(httpClient);
     }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
index 717d3a8da5352..de47395f0a9eb 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
@@ -384,7 +384,13 @@ private List<NamedExpression> expandProjections(List<? extends NamedExpression>
             List<Attribute> output = child.output();
             for (NamedExpression ne : projections) {
                 if (ne instanceof UnresolvedStar) {
-                    result.addAll(expandStar((UnresolvedStar) ne, output));
+                    List<NamedExpression> expanded = expandStar((UnresolvedStar) ne, output);
+                    // the field exists, but cannot be expanded (no sub-fields)
+                    if (expanded.isEmpty()) {
+                        result.add(ne);
+                    } else {
+                        result.addAll(expanded);
+                    }
                 } else if (ne instanceof UnresolvedAlias) {
                     UnresolvedAlias ua = (UnresolvedAlias) ne;
                     if (ua.child() instanceof UnresolvedStar) {
@@ -407,6 +413,13 @@ private List<NamedExpression> expandStar(UnresolvedStar us, List<Attribute> outp
                 // since this is an unresolved start we don't know whether it's a path or an actual qualifier
                 Attribute q = resolveAgainstList(us.qualifier(), output);
 
+                // the wildcard couldn't be expanded because the field doesn't exist at all
+                // so, add to the list of expanded attributes its qualifier (the field without the wildcard)
+                // the qualifier will be unresolved and later used in the error message presented to the user
+                if (q == null) {
+                    expanded.add(us.qualifier());
+                    return expanded;
+                }
                 // now use the resolved 'qualifier' to match
                 for (Attribute attr : output) {
                     // filter the attributes that match based on their path
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java
index c8834240c6ceb..5394625d88272 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.sql.expression.function.Functions;
 import org.elasticsearch.xpack.sql.expression.function.Score;
 import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.sql.plan.logical.Distinct;
 import org.elasticsearch.xpack.sql.plan.logical.Filter;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
index 2a30db7e44d7b..94ea45b5ec83e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.execution.search;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequest;
@@ -16,7 +17,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
@@ -42,7 +42,7 @@
  */
 public class CompositeAggregationCursor implements Cursor {
 
-    private final Logger log = Loggers.getLogger(getClass());
+    private final Logger log = LogManager.getLogger(getClass());
 
     public static final String NAME = "c";
 
@@ -229,4 +229,4 @@ public boolean equals(Object obj) {
     public String toString() {
         return "cursor for composite on index [" + Arrays.toString(indices) + "]";
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
index e3c604feda076..f3a397dc68b3d 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.execution.search;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequest;
@@ -13,7 +14,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -61,7 +61,7 @@
 // TODO: add retry/back-off
 public class Querier {
 
-    private final Logger log = Loggers.getLogger(getClass());
+    private final Logger log = LogManager.getLogger(getClass());
 
     private final TimeValue keepAlive, timeout;
     private final int size;
@@ -443,4 +443,4 @@ public final void onFailure(Exception ex) {
             listener.onFailure(ex);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java
index 0fa20022f1cd3..dcaca223f4ce2 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/ScrollCursor.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.execution.search;
 
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.ClearScrollRequest;
@@ -16,7 +17,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.sql.execution.search.extractor.HitExtractor;
 import org.elasticsearch.xpack.sql.session.Configuration;
 import org.elasticsearch.xpack.sql.session.Cursor;
@@ -28,7 +28,7 @@
 
 public class ScrollCursor implements Cursor {
 
-    private final Logger log = Loggers.getLogger(getClass());
+    private final Logger log = LogManager.getLogger(getClass());
 
     public static final String NAME = "s";
 
@@ -124,4 +124,4 @@ public static void cleanCursor(Client client, String scrollId, ActionListener<Cl
         clearScrollRequest.addScrollId(scrollId);
         client.clearScroll(clearScrollRequest, listener);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSet.java
index 417a3cf3f4916..e8994bf108f8e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSet.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitRowSet.java
@@ -44,9 +44,11 @@ class SearchHitRowSet extends AbstractRowSet {
 
         String innerHit = null;
         for (HitExtractor ex : exts) {
-            innerHit = ex.hitName();
-            if (innerHit != null) {
-                innerHits.add(innerHit);
+            if (ex.hitName() != null) {
+                innerHits.add(ex.hitName());
+                if (innerHit == null) {
+                    innerHit = ex.hitName();
+                }
             }
         }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java
index 4a29358a7fa61..204ea8b11023e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java
@@ -78,6 +78,7 @@ public Object fold() {
         throw new SqlIllegalArgumentException("Should not fold expression");
     }
 
+    // whether the expression becomes null if at least one param/input is null
     public abstract boolean nullable();
 
     // the references/inputs/leaves of the expression tree
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java
index 5a35e00ab9d6c..92f94140fa0f8 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java
@@ -18,6 +18,7 @@
 import java.util.Locale;
 import java.util.function.Predicate;
 
+import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 
@@ -103,6 +104,10 @@ public static String name(Expression e) {
         return e instanceof NamedExpression ? ((NamedExpression) e).name() : e.nodeName();
     }
 
+    public static boolean isNull(Expression e) {
+        return e.dataType() == DataType.NULL || (e.foldable() && e.fold() == null);
+    }
+
     public static List<String> names(Collection<? extends Expression> e) {
         List<String> names = new ArrayList<>(e.size());
         for (Expression ex : e) {
@@ -137,6 +142,14 @@ public static Pipe pipe(Expression e) {
         throw new SqlIllegalArgumentException("Cannot create pipe for {}", e);
     }
 
+    public static List<Pipe> pipe(List<Expression> expressions) {
+        List<Pipe> pipes = new ArrayList<>(expressions.size());
+        for (Expression e : expressions) {
+            pipes.add(pipe(e));
+        }
+        return pipes;
+    }
+
     public static TypeResolution typeMustBeBoolean(Expression e, String operationName, ParamOrdinal paramOrd) {
         return typeMustBe(e, dt -> dt == DataType.BOOLEAN, operationName, paramOrd, "boolean");
     }
@@ -161,27 +174,18 @@ public static TypeResolution typeMustBeNumericOrDate(Expression e, String operat
         return typeMustBe(e, dt -> dt.isNumeric() || dt == DataType.DATE, operationName, paramOrd, "numeric", "date");
     }
 
-    private static TypeResolution typeMustBe(Expression e,
+    public static TypeResolution typeMustBe(Expression e,
                                              Predicate<DataType> predicate,
                                              String operationName,
-                                             ParamOrdinal pOrd,
+                                             ParamOrdinal paramOrd,
                                              String... acceptedTypes) {
-
         return predicate.test(e.dataType()) || DataTypes.isNull(e.dataType())?
             TypeResolution.TYPE_RESOLVED :
-            new TypeResolution(incorrectTypeErrorMessage(e, operationName, pOrd, acceptedTypes));
-
-    }
-
-    private static String incorrectTypeErrorMessage(Expression e,
-                                                    String operationName,
-                                                    ParamOrdinal paramOrd,
-                                                    String... acceptedTypes) {
-        return String.format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]",
-            operationName,
-            paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
-            Strings.arrayToDelimitedString(acceptedTypes, " or "),
-            Expressions.name(e),
-            e.dataType().esType);
-    }
-}
+            new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]",
+                    operationName,
+                    paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
+                    Strings.arrayToDelimitedString(acceptedTypes, " or "),
+                    Expressions.name(e),
+                    e.dataType().esType));
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java
index 2e44240cc0cbe..7148a08a3facd 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java
@@ -161,7 +161,11 @@ public static Literal of(String name, Expression foldable) {
         if (name == null) {
             name = foldable instanceof NamedExpression ? ((NamedExpression) foldable).name() : String.valueOf(fold);
         }
-
         return new Literal(foldable.location(), name, fold, foldable.dataType());
     }
-}
+
+    public static Literal of(Expression source, Object value) {
+        String name = source instanceof NamedExpression ? ((NamedExpression) source).name() : String.valueOf(value);
+        return new Literal(source.location(), name, value, source.dataType());
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnresolvedStar.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnresolvedStar.java
index 948a5465efa80..c9ef08eab5aee 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnresolvedStar.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/UnresolvedStar.java
@@ -66,12 +66,12 @@ public boolean equals(Object obj) {
     }
 
     private String message() {
-        return (qualifier() != null ? qualifier() + "." : "") + "*";
+        return (qualifier() != null ? qualifier().qualifiedName() + "." : "") + "*";
     }
 
     @Override
     public String unresolvedMessage() {
-        return "Cannot determine columns for " + message();
+        return "Cannot determine columns for [" + message() + "]";
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java
index 6c6f1a2633ac8..000a9c097c98f 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/Function.java
@@ -46,7 +46,7 @@ public String name() {
 
     @Override
     public boolean nullable() {
-        return false;
+        return Expressions.nullable(children());
     }
 
     @Override
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
index b36789bfbc6be..70c9c18a5a4c0 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
@@ -82,6 +82,7 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.Space;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.Substring;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.UCase;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.Coalesce;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mod;
 import org.elasticsearch.xpack.sql.parser.ParsingException;
 import org.elasticsearch.xpack.sql.tree.Location;
@@ -142,6 +143,8 @@ private void defineDefaultFunctions() {
                 def(Skewness.class, Skewness::new),
                 def(Kurtosis.class, Kurtosis::new));
         // Scalar functions
+        // conditional
+        addToMap(def(Coalesce.class, Coalesce::new));
         // Date
         addToMap(def(DayName.class, DayName::new, "DAYNAME"),
                 def(DayOfMonth.class, DayOfMonth::new, "DAYOFMONTH", "DAY", "DOM"),
@@ -310,6 +313,26 @@ static <T extends Function> FunctionDefinition def(Class<T> function,
         return def(function, builder, false, aliases);
     }
 
+    /**
+     * Build a {@linkplain FunctionDefinition} for multi-arg function that
+     * is not aware of time zone and does not support {@code DISTINCT}.
+     */
+    @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do
+    static <T extends Function> FunctionDefinition def(Class<T> function,
+            MultiFunctionBuilder<T> ctorRef, String... aliases) {
+        FunctionBuilder builder = (location, children, distinct, tz) -> {
+            if (distinct) {
+                throw new IllegalArgumentException("does not support DISTINCT yet it was specified");
+            }
+            return ctorRef.build(location, children);
+        };
+        return def(function, builder, false, aliases);
+    }
+
+    interface MultiFunctionBuilder<T> {
+        T build(Location location, List<Expression> children);
+    }
+    
     /**
      * Build a {@linkplain FunctionDefinition} for a unary function that is not
      * aware of time zone but does support {@code DISTINCT}.
@@ -325,6 +348,7 @@ static <T extends Function> FunctionDefinition def(Class<T> function,
         };
         return def(function, builder, false, aliases);
     }
+
     interface DistinctAwareUnaryFunctionBuilder<T> {
         T build(Location location, Expression target, boolean distinct);
     }
@@ -347,6 +371,7 @@ static <T extends Function> FunctionDefinition def(Class<T> function,
         };
         return def(function, builder, true, aliases);
     }
+
     interface DatetimeUnaryFunctionBuilder<T> {
         T build(Location location, Expression target, TimeZone tz);
     }
@@ -373,6 +398,7 @@ static <T extends Function> FunctionDefinition def(Class<T> function,
         };
         return def(function, builder, false, aliases);
     }
+
     interface BinaryFunctionBuilder<T> {
         T build(Location location, Expression lhs, Expression rhs);
     }
@@ -391,6 +417,7 @@ private static FunctionDefinition def(Class<? extends Function> function, Functi
         };
         return new FunctionDefinition(primaryName, unmodifiableList(Arrays.asList(aliases)), function, datetime, realBuilder);
     }
+
     private interface FunctionBuilder {
         Function build(Location location, List<Expression> children, boolean distinct, TimeZone tz);
     }
@@ -450,6 +477,7 @@ private static <T extends Function> FunctionDefinition def(Class<T> function,
             ctorRef.build(location, children.get(0), children.get(0).dataType());
         return def(function, builder, false, aliases);
     }
+
     private interface CastFunctionBuilder<T> {
         T build(Location location, Expression expression, DataType dataType);
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java
index 5ed81e354fce2..b2f4ab8ef2ca2 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionType.java
@@ -8,11 +8,13 @@
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.ConditionalFunction;
 
 
 public enum FunctionType {
 
     AGGREGATE(AggregateFunction.class),
+    CONDITIONAL(ConditionalFunction.class),
     SCALAR(ScalarFunction.class),
     SCORE(Score.class);
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
index ae35f9c760c43..6954abbbaa7dc 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java
@@ -25,9 +25,15 @@
 import org.elasticsearch.xpack.sql.expression.gen.processor.ConstantProcessor;
 import org.elasticsearch.xpack.sql.expression.gen.processor.HitExtractorProcessor;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.CoalesceProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.logical.BinaryLogicProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.logical.NotProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNullProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparisonProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -49,13 +55,24 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(new Entry(Processor.class, CastProcessor.NAME, CastProcessor::new));
         entries.add(new Entry(Processor.class, ChainingProcessor.NAME, ChainingProcessor::new));
 
-        // comparators
-        entries.add(new Entry(Processor.class, BinaryComparisonProcessor.NAME, BinaryComparisonProcessor::new));
+        // logical
+        entries.add(new Entry(Processor.class, BinaryLogicProcessor.NAME, BinaryLogicProcessor::new));
+        entries.add(new Entry(Processor.class, NotProcessor.NAME, NotProcessor::new));
+        // null
+        entries.add(new Entry(Processor.class, CoalesceProcessor.NAME, CoalesceProcessor::new));
+        entries.add(new Entry(Processor.class, IsNotNullProcessor.NAME, IsNotNullProcessor::new));
 
         // arithmetic
         entries.add(new Entry(Processor.class, BinaryArithmeticProcessor.NAME, BinaryArithmeticProcessor::new));
         entries.add(new Entry(Processor.class, UnaryArithmeticProcessor.NAME, UnaryArithmeticProcessor::new));
         entries.add(new Entry(Processor.class, BinaryMathProcessor.NAME, BinaryMathProcessor::new));
+        // comparators
+        entries.add(new Entry(Processor.class, BinaryComparisonProcessor.NAME, BinaryComparisonProcessor::new));
+        entries.add(new Entry(Processor.class, InProcessor.NAME, InProcessor::new));
+        // regex
+        entries.add(new Entry(Processor.class, RegexProcessor.NAME, RegexProcessor::new));
+
+
         // datetime
         entries.add(new Entry(Processor.class, DateTimeProcessor.NAME, DateTimeProcessor::new));
         entries.add(new Entry(Processor.class, NamedDateTimeProcessor.NAME, NamedDateTimeProcessor::new));
@@ -73,4 +90,12 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         entries.add(new Entry(Processor.class, SubstringFunctionProcessor.NAME, SubstringFunctionProcessor::new));
         return entries;
     }
-}
\ No newline at end of file
+
+    public static List<Object> process(List<Processor> processors, Object input) {
+        List<Object> values = new ArrayList<>(processors.size());
+        for (Processor p : processors) {
+            values.add(p.process(input));
+        }
+        return values;
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
index c2c8177cfb37d..3bd03986eb5c4 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Concat.java
@@ -49,6 +49,11 @@ protected Pipe makePipe() {
         return new ConcatFunctionPipe(location(), this, Expressions.pipe(left()), Expressions.pipe(right()));
     }
     
+    @Override
+    public boolean nullable() {
+        return left().nullable() && right().nullable();
+    }
+
     @Override
     public boolean foldable() {
         return left().foldable() && right().foldable();
@@ -80,4 +85,4 @@ public ScriptTemplate scriptWithField(FieldAttribute field) {
     public DataType dataType() {
         return DataType.KEYWORD;
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
index 9aabb3f10ecdc..beeb1174ba245 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
@@ -21,16 +21,19 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.ReplaceFunctionProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.SubstringFunctionProcessor;
-import org.elasticsearch.xpack.sql.expression.predicate.IsNotNullProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.CoalesceProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.NotProcessor;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNullProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InProcessor;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation;
 import org.elasticsearch.xpack.sql.util.StringUtils;
 
 import java.time.ZonedDateTime;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -38,6 +41,7 @@
  * Acts as a registry of the various static methods used <b>internally</b> by the scalar functions
  * (to simplify the whitelist definition).
  */
+@SuppressWarnings("unused")
 public final class InternalSqlScriptUtils {
 
     private InternalSqlScriptUtils() {}
@@ -50,7 +54,7 @@ private InternalSqlScriptUtils() {}
     public static <T> Object docValue(Map<String, ScriptDocValues<T>> doc, String fieldName) {
         if (doc.containsKey(fieldName)) {
             ScriptDocValues<T> docValues = doc.get(fieldName);
-            if (docValues.size() > 0) {
+            if (!docValues.isEmpty()) {
                 return docValues.get(0);
             }
         }
@@ -81,6 +85,10 @@ public static Boolean eq(Object left, Object right) {
         return BinaryComparisonOperation.EQ.apply(left, right);
     }
 
+    public static Boolean neq(Object left, Object right) {
+        return BinaryComparisonOperation.NEQ.apply(left, right);
+    }
+
     public static Boolean lt(Object left, Object right) {
         return BinaryComparisonOperation.LT.apply(left, right);
     }
@@ -113,6 +121,17 @@ public static Boolean notNull(Object expression) {
         return IsNotNullProcessor.apply(expression);
     }
 
+    public static Boolean in(Object value, List<Object> values) {
+        return InProcessor.apply(value, values);
+    }
+
+    //
+    // Null
+    //
+    public static Object coalesce(List<Object> expressions) {
+        return CoalesceProcessor.apply(expressions);
+    }
+
     //
     // Regex
     //
@@ -375,4 +394,4 @@ public static String substring(String s, Number start, Number length) {
     public static String ucase(String s) {
         return (String) StringOperation.UCASE.apply(s);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/MultiPipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/MultiPipe.java
new file mode 100644
index 0000000000000..d25e7a2e660a2
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/MultiPipe.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class MultiPipe extends Pipe {
+
+    protected MultiPipe(Location location, Expression expression, List<Pipe> children) {
+        super(location, expression, children);
+    }
+
+    @Override
+    public Processor asProcessor() {
+        List<Processor> procs = new ArrayList<>();
+        for (Pipe pipe : children()) {
+            procs.add(pipe.asProcessor());
+        }
+
+        return asProcessor(procs);
+    }
+
+    public abstract Processor asProcessor(List<Processor> procs);
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/Pipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/Pipe.java
index 5c96d2c9244ab..b92cb9a15eae4 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/Pipe.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/Pipe.java
@@ -6,13 +6,16 @@
 package org.elasticsearch.xpack.sql.expression.gen.pipeline;
 
 import org.elasticsearch.xpack.sql.capabilities.Resolvable;
+import org.elasticsearch.xpack.sql.capabilities.Resolvables;
 import org.elasticsearch.xpack.sql.execution.search.FieldExtraction;
+import org.elasticsearch.xpack.sql.execution.search.SqlSourceBuilder;
 import org.elasticsearch.xpack.sql.expression.Attribute;
 import org.elasticsearch.xpack.sql.expression.Expression;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
 import org.elasticsearch.xpack.sql.tree.Location;
 import org.elasticsearch.xpack.sql.tree.Node;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -38,6 +41,27 @@ public Expression expression() {
         return expression;
     }
 
+    @Override
+    public boolean resolved() {
+        return Resolvables.resolved(children());
+    }
+
+    @Override
+    public void collectFields(SqlSourceBuilder sourceBuilder) {
+        children().forEach(c -> c.collectFields(sourceBuilder));
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        for (Pipe pipe : children()) {
+            if (pipe.supportedByAggsOnlyQuery()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     public abstract Processor asProcessor();
 
     /**
@@ -47,9 +71,16 @@ public Expression expression() {
      * @return {@code this} if the resolution doesn't change the
      *      definition, a new {@link Pipe} otherwise
      */
-    public abstract Pipe resolveAttributes(AttributeResolver resolver);
+    public Pipe resolveAttributes(AttributeResolver resolver) {
+        List<Pipe> newPipes = new ArrayList<>(children().size());
+        for (Pipe p : children()) {
+            newPipes.add(p.resolveAttributes(resolver));
+        }
+
+        return children().equals(newPipes) ? this : replaceChildren(newPipes);
+    }
 
     public interface AttributeResolver {
         FieldExtraction resolve(Attribute attribute);
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/UnaryPipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/UnaryPipe.java
index 8e36f448929ab..8e2c87dc75cda 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/UnaryPipe.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/pipeline/UnaryPipe.java
@@ -98,4 +98,4 @@ public boolean equals(Object obj) {
                 && Objects.equals(child, other.child)
                 && Objects.equals(expression(), other.expression());
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/Scripts.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/Scripts.java
index f9e2588a9c035..21ac12e51da89 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/Scripts.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/Scripts.java
@@ -87,4 +87,4 @@ public static ScriptTemplate binaryMethod(String methodName, ScriptTemplate left
                     .build(),
                 dataType);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Coalesce.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Coalesce.java
new file mode 100644
index 0000000000000..8a2ddc4d10564
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/Coalesce.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.predicate.conditional;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.Expressions;
+import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder;
+import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+import org.elasticsearch.xpack.sql.type.DataType;
+import org.elasticsearch.xpack.sql.type.DataTypeConversion;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import static org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+public class Coalesce extends ConditionalFunction {
+
+    private DataType dataType = DataType.NULL;
+
+    public Coalesce(Location location, List<Expression> fields) {
+        super(location, fields);
+    }
+
+    @Override
+    protected NodeInfo<Coalesce> info() {
+        return NodeInfo.create(this, Coalesce::new, children());
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Coalesce(location(), newChildren);
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        for (Expression e : children()) {
+            dataType = DataTypeConversion.commonType(dataType, e.dataType());
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    @Override
+    public DataType dataType() {
+        return dataType;
+    }
+
+    @Override
+    public boolean foldable() {
+        // if the first entry is foldable, so is coalesce
+        // that's because the nulls are eliminated by the optimizer
+        // and if the first expression is folded (and not null), the rest do not matter
+        List<Expression> children = children();
+        return (children.isEmpty() || (children.get(0).foldable() && children.get(0).fold() != null));
+    }
+
+    @Override
+    public Object fold() {
+        List<Expression> children = children();
+        return children.isEmpty() ? null : children.get(0).fold();
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        List<ScriptTemplate> templates = new ArrayList<>();
+        for (Expression ex : children()) {
+            templates.add(asScript(ex));
+        }
+
+        StringJoiner template = new StringJoiner(",", "{sql}.coalesce([", "])");
+        ParamsBuilder params = paramsBuilder();
+
+        for (ScriptTemplate scriptTemplate : templates) {
+            template.add(scriptTemplate.template());
+            params.script(scriptTemplate.params());
+        }
+
+        return new ScriptTemplate(template.toString(), params.build(), dataType);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new CoalescePipe(location(), this, Expressions.pipe(children()));
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalescePipe.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalescePipe.java
new file mode 100644
index 0000000000000..a19d13e02df92
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalescePipe.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.predicate.conditional;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.gen.pipeline.MultiPipe;
+import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+import java.util.List;
+
+public class CoalescePipe extends MultiPipe {
+
+    public CoalescePipe(Location location, Expression expression, List<Pipe> children) {
+        super(location, expression, children);
+    }
+
+    @Override
+    protected NodeInfo<CoalescePipe> info() {
+        return NodeInfo.create(this, CoalescePipe::new, expression(), children());
+    }
+
+    @Override
+    public Pipe replaceChildren(List<Pipe> newChildren) {
+        return new CoalescePipe(location(), expression(), newChildren);
+    }
+
+    @Override
+    public Processor asProcessor(List<Processor> procs) {
+        return new CoalesceProcessor(procs);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalesceProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalesceProcessor.java
new file mode 100644
index 0000000000000..e23b43634238d
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CoalesceProcessor.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.predicate.conditional;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class CoalesceProcessor implements Processor {
+    
+    public static final String NAME = "nco";
+
+    private final List<Processor> processsors;
+
+    public CoalesceProcessor(List<Processor> processors) {
+        this.processsors = processors;
+    }
+
+    public CoalesceProcessor(StreamInput in) throws IOException {
+        processsors = in.readNamedWriteableList(Processor.class);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteableList(processsors);
+    }
+
+    @Override
+    public Object process(Object input) {
+        for (Processor proc : processsors) {
+            Object result = proc.process(input);
+            if (result != null) {
+                return result;
+            }
+        }
+        return null;
+    }
+
+    public static Object apply(List<?> values) {
+        if (values == null || values.isEmpty()) {
+            return null;
+        }
+
+        for (Object object : values) {
+            if (object != null) {
+                return object;
+            }
+        }
+
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(processsors);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        CoalesceProcessor that = (CoalesceProcessor) o;
+        return Objects.equals(processsors, that.processsors);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/ConditionalFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/ConditionalFunction.java
new file mode 100644
index 0000000000000..2feff40643335
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/ConditionalFunction.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression.predicate.conditional;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.sql.tree.Location;
+
+import java.util.List;
+
+/**
+ * Base class for conditional predicates.
+ */
+public abstract class ConditionalFunction extends ScalarFunction {
+
+    protected ConditionalFunction(Location location, List<Expression> fields) {
+        super(location, fields);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogic.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogic.java
index be61b1906d588..93c50fbc1351b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogic.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogic.java
@@ -33,4 +33,9 @@ protected TypeResolution resolveInputType(Expression e, Expressions.ParamOrdinal
     protected Pipe makePipe() {
         return new BinaryLogicPipe(location(), this, Expressions.pipe(left()), Expressions.pipe(right()), function());
     }
+
+    @Override
+    public boolean nullable() {
+        return left().nullable() && right().nullable();
+    }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessor.java
index 334a80b7d578d..15f024b4f539e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessor.java
@@ -27,7 +27,7 @@ public enum BinaryLogicOperation implements PredicateBiFunction<Boolean, Boolean
                 return null;
             }
             return Boolean.logicalAnd(l.booleanValue(), r.booleanValue());
-        }, "&&"),
+        }, "AND"),
         OR((l, r) -> {
             if (Boolean.TRUE.equals(l) || Boolean.TRUE.equals(r)) {
                 return Boolean.TRUE;
@@ -36,7 +36,7 @@ public enum BinaryLogicOperation implements PredicateBiFunction<Boolean, Boolean
                 return null;
             }
             return Boolean.logicalOr(l.booleanValue(), r.booleanValue());
-        }, "||");
+        }, "OR");
 
         private final BiFunction<Boolean, Boolean, Boolean> process;
         private final String symbol;
@@ -84,8 +84,18 @@ public String getWriteableName() {
 
     @Override
     protected void checkParameter(Object param) {
-        if (!(param instanceof Boolean)) {
+        if (param != null && !(param instanceof Boolean)) {
             throw new SqlIllegalArgumentException("A boolean is required; received {}", param);
         }
     }
-}
\ No newline at end of file
+
+    @Override
+    public Object process(Object input) {
+        Object l = left().process(input);
+        checkParameter(l);
+        Object r = right().process(input);
+        checkParameter(r);
+
+        return doProcess(l, r);
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/NotProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/NotProcessor.java
index 14425d35578ac..3480854d5015e 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/NotProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/logical/NotProcessor.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.expression.predicate.logical;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
@@ -13,12 +14,14 @@
 
 public class NotProcessor implements Processor {
     
-    static final NotProcessor INSTANCE = new NotProcessor();
+    public static final NotProcessor INSTANCE = new NotProcessor();
 
     public static final String NAME = "ln";
 
     private NotProcessor() {}
 
+    public NotProcessor(StreamInput in) throws IOException {}
+
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNull.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNull.java
similarity index 96%
rename from x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNull.java
rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNull.java
index bd3fd5bf0811b..c4c848aecff32 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNull.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNull.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.sql.expression.predicate;
+package org.elasticsearch.xpack.sql.expression.predicate.nulls;
 
 import org.elasticsearch.xpack.sql.expression.Expression;
 import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNullProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNullProcessor.java
similarity index 83%
rename from x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNullProcessor.java
rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNullProcessor.java
index b29ae263f3907..ddc5f11975165 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/IsNotNullProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/nulls/IsNotNullProcessor.java
@@ -3,8 +3,9 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.sql.expression.predicate;
+package org.elasticsearch.xpack.sql.expression.predicate.nulls;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
 
@@ -14,10 +15,12 @@ public class IsNotNullProcessor implements Processor {
     
     static final IsNotNullProcessor INSTANCE = new IsNotNullProcessor();
 
-    public static final String NAME = "inn";
+    public static final String NAME = "ninn";
 
     private IsNotNullProcessor() {}
 
+    public IsNotNullProcessor(StreamInput in) throws IOException {}
+
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessor.java
index e33d7b282006d..7c7983cf2c1bb 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessor.java
@@ -19,6 +19,7 @@ public class BinaryComparisonProcessor extends FunctionalBinaryProcessor<Object,
     public enum BinaryComparisonOperation implements PredicateBiFunction<Object, Object, Boolean> {
 
         EQ(Comparisons::eq, "=="),
+        NEQ(Comparisons::neq, "!="),
         GT(Comparisons::gt, ">"),
         GTE(Comparisons::gte, ">="),
         LT(Comparisons::lt, "<"),
@@ -62,4 +63,4 @@ public BinaryComparisonProcessor(StreamInput in) throws IOException {
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Comparisons.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Comparisons.java
index 79d3f2b318b59..7c45371a24848 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Comparisons.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Comparisons.java
@@ -19,6 +19,11 @@ public static Boolean eq(Object l, Object r) {
         return i == null ? null : i.intValue() == 0;
     }
 
+    static Boolean neq(Object l, Object r) {
+        Integer i = compare(l, r);
+        return i == null ? null : i.intValue() != 0;
+    }
+
     static Boolean lt(Object l, Object r) {
         Integer i = compare(l, r);
         return i == null ? null : i.intValue() < 0;
@@ -50,6 +55,9 @@ static Boolean in(Object l, Set<Object> r) {
      */
     @SuppressWarnings({ "rawtypes", "unchecked" })
     static Integer compare(Object l, Object r) {
+        if (l == null || r == null) {
+            return null;
+        }
         // typical number comparison
         if (l instanceof Number && r instanceof Number) {
             return compare((Number) l, (Number) r);
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Equals.java
index 15dbacafc4add..23b8b879123b3 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Equals.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/Equals.java
@@ -6,11 +6,12 @@
 package org.elasticsearch.xpack.sql.expression.predicate.operator.comparison;
 
 import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.predicate.BinaryOperator;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
 import org.elasticsearch.xpack.sql.tree.Location;
 import org.elasticsearch.xpack.sql.tree.NodeInfo;
 
-public class Equals extends BinaryComparison {
+public class Equals extends BinaryComparison implements BinaryOperator.Negateable {
 
     public Equals(Location location, Expression left, Expression right) {
         super(location, left, right, BinaryComparisonOperation.EQ);
@@ -30,4 +31,9 @@ protected Equals replaceChildren(Expression newLeft, Expression newRight) {
     public Equals swapLeftAndRight() {
         return new Equals(location(), right(), left());
     }
+
+    @Override
+    public BinaryOperator<?, ?, ?, ?> negate() {
+        return new NotEquals(location(), left(), right());
+    }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/In.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/In.java
similarity index 62%
rename from x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/In.java
rename to x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/In.java
index 9b16b77511ca7..8a3ac8edaee77 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/In.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/In.java
@@ -3,20 +3,17 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.sql.expression.predicate;
+package org.elasticsearch.xpack.sql.expression.predicate.operator.comparison;
 
 import org.elasticsearch.xpack.sql.expression.Attribute;
 import org.elasticsearch.xpack.sql.expression.Expression;
 import org.elasticsearch.xpack.sql.expression.Expressions;
+import org.elasticsearch.xpack.sql.expression.Foldables;
 import org.elasticsearch.xpack.sql.expression.NamedExpression;
 import org.elasticsearch.xpack.sql.expression.function.scalar.ScalarFunctionAttribute;
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
-import org.elasticsearch.xpack.sql.expression.gen.script.Params;
-import org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder;
 import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate;
 import org.elasticsearch.xpack.sql.expression.gen.script.ScriptWeaver;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Comparisons;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InPipe;
 import org.elasticsearch.xpack.sql.tree.Location;
 import org.elasticsearch.xpack.sql.tree.NodeInfo;
 import org.elasticsearch.xpack.sql.type.DataType;
@@ -30,7 +27,6 @@
 import java.util.StringJoiner;
 import java.util.stream.Collectors;
 
-import static java.lang.String.format;
 import static org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder.paramsBuilder;
 
 public class In extends NamedExpression implements ScriptWeaver {
@@ -84,29 +80,17 @@ public boolean foldable() {
 
     @Override
     public Boolean fold() {
-        if (value.dataType() == DataType.NULL) {
+        // Optimization for early return and Query folding to LocalExec
+        if (value.dataType() == DataType.NULL ||
+            list.size() == 1 && list.get(0).dataType() == DataType.NULL) {
             return null;
         }
-        if (list.size() == 1 && list.get(0).dataType() == DataType.NULL) {
-            return false;
-        }
-
-        Object foldedLeftValue = value.fold();
-        Boolean result = false;
-        for (Expression rightValue : list) {
-            Boolean compResult = Comparisons.eq(foldedLeftValue, rightValue.fold());
-            if (compResult == null) {
-                result = null;
-            } else if (compResult) {
-                return true;
-            }
-        }
-        return result;
+        return InProcessor.apply(value.fold(), Foldables.valuesOf(list, value.dataType()));
     }
 
     @Override
     public String name() {
-        StringJoiner sj = new StringJoiner(", ", " IN(", ")");
+        StringJoiner sj = new StringJoiner(", ", " IN (", ")");
         list.forEach(e -> sj.add(Expressions.name(e)));
         return Expressions.name(value) + sj.toString();
     }
@@ -122,34 +106,18 @@ public Attribute toAttribute() {
 
     @Override
     public ScriptTemplate asScript() {
-        StringJoiner sj = new StringJoiner(" || ");
         ScriptTemplate leftScript = asScript(value);
-        List<Params> rightParams = new ArrayList<>();
-        String scriptPrefix = leftScript + "==";
-        LinkedHashSet<Object> values = list.stream().map(Expression::fold).collect(Collectors.toCollection(LinkedHashSet::new));
-        for (Object valueFromList : values) {
-            // if checked against null => false
-            if (valueFromList != null) {
-                if (valueFromList instanceof Expression) {
-                    ScriptTemplate rightScript = asScript((Expression) valueFromList);
-                    sj.add(scriptPrefix + rightScript.template());
-                    rightParams.add(rightScript.params());
-                } else {
-                    if (valueFromList instanceof String) {
-                        sj.add(scriptPrefix + '"' + valueFromList + '"');
-                    } else {
-                        sj.add(scriptPrefix + valueFromList.toString());
-                    }
-                }
-            }
-        }
 
-        ParamsBuilder paramsBuilder = paramsBuilder().script(leftScript.params());
-        for (Params p : rightParams) {
-            paramsBuilder = paramsBuilder.script(p);
-        }
+        // fold & remove duplicates
+        List<Object> values = new ArrayList<>(new LinkedHashSet<>(Foldables.valuesOf(list, value.dataType())));
 
-        return new ScriptTemplate(format(Locale.ROOT, "%s", sj.toString()), paramsBuilder.build(), dataType());
+        return new ScriptTemplate(
+            formatTemplate(String.format(Locale.ROOT, "{sql}.in(%s, {})", leftScript.template())),
+            paramsBuilder()
+                .script(leftScript.params())
+                .variable(values)
+                .build(),
+            dataType());
     }
 
     @Override
@@ -173,6 +141,6 @@ public boolean equals(Object obj) {
 
         In other = (In) obj;
         return Objects.equals(value, other.value)
-                && Objects.equals(list, other.list);
+            && Objects.equals(list, other.list);
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessor.java
index 0a901b5b5e6fe..82233e250e364 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessor.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
 
 import java.io.IOException;
@@ -19,7 +20,7 @@ public class InProcessor implements Processor {
 
     private final List<Processor> processsors;
 
-    public InProcessor(List<Processor> processors) {
+    InProcessor(List<Processor> processors) {
         this.processsors = processors;
     }
 
@@ -40,14 +41,17 @@ public final void writeTo(StreamOutput out) throws IOException {
     @Override
     public Object process(Object input) {
         Object leftValue = processsors.get(processsors.size() - 1).process(input);
-        Boolean result = false;
+        return apply(leftValue, Processors.process(processsors.subList(0, processsors.size() - 1), leftValue));
+    }
 
-        for (int i = 0; i < processsors.size() - 1; i++) {
-            Boolean compResult = Comparisons.eq(leftValue, processsors.get(i).process(input));
+    public static Boolean apply(Object input, List<Object> values) {
+        Boolean result = Boolean.FALSE;
+        for (Object v : values) {
+            Boolean compResult = Comparisons.eq(input, v);
             if (compResult == null) {
                 result = null;
-            } else if (compResult) {
-                return true;
+            } else if (compResult == Boolean.TRUE) {
+                return Boolean.TRUE;
             }
         }
         return result;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/NotEquals.java
new file mode 100644
index 0000000000000..41e0a939fbd07
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/NotEquals.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.predicate.operator.comparison;
+
+import org.elasticsearch.xpack.sql.expression.Expression;
+import org.elasticsearch.xpack.sql.expression.predicate.BinaryOperator;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
+import org.elasticsearch.xpack.sql.tree.Location;
+import org.elasticsearch.xpack.sql.tree.NodeInfo;
+
+public class NotEquals extends BinaryComparison implements BinaryOperator.Negateable {
+
+    public NotEquals(Location location, Expression left, Expression right) {
+        super(location, left, right, BinaryComparisonOperation.NEQ);
+    }
+
+    @Override
+    protected NodeInfo<NotEquals> info() {
+        return NodeInfo.create(this, NotEquals::new, left(), right());
+    }
+
+    @Override
+    protected NotEquals replaceChildren(Expression newLeft, Expression newRight) {
+        return new NotEquals(location(), newLeft, newRight);
+    }
+
+    @Override
+    public NotEquals swapLeftAndRight() {
+        return new NotEquals(location(), right(), left());
+    }
+
+    @Override
+    public BinaryOperator<?, ?, ?, ?> negate() {
+        return new Equals(location(), left(), right());
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
index 8443358a12cb2..039f25b5ef590 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
@@ -41,15 +41,19 @@
 import org.elasticsearch.xpack.sql.expression.predicate.BinaryPredicate;
 import org.elasticsearch.xpack.sql.expression.predicate.Predicates;
 import org.elasticsearch.xpack.sql.expression.predicate.Range;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.Coalesce;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.sql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.sql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.sql.plan.logical.Filter;
@@ -63,6 +67,7 @@
 import org.elasticsearch.xpack.sql.rule.RuleExecutor;
 import org.elasticsearch.xpack.sql.session.EmptyExecutable;
 import org.elasticsearch.xpack.sql.session.SingletonExecutable;
+import org.elasticsearch.xpack.sql.type.DataType;
 import org.elasticsearch.xpack.sql.util.CollectionUtils;
 
 import java.util.ArrayList;
@@ -122,7 +127,9 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
                 new CombineProjections(),
                 // folding
                 new ReplaceFoldableAttributes(),
+                new FoldNull(),
                 new ConstantFolding(),
+                new SimplifyCoalesce(),
                 // boolean
                 new BooleanSimplification(),
                 new BooleanLiteralsOnTheRight(),
@@ -682,8 +689,7 @@ protected LogicalPlan rule(Filter filter) {
                 if (TRUE.equals(filter.condition())) {
                     return filter.child();
                 }
-                // TODO: add comparison with null as well
-                if (FALSE.equals(filter.condition())) {
+                if (FALSE.equals(filter.condition()) || Expressions.isNull(filter.condition())) {
                     return new LocalRelation(filter.location(), new EmptyExecutable(filter.output()));
                 }
             }
@@ -1112,6 +1118,37 @@ private boolean canPropagateFoldable(LogicalPlan p) {
         }
     }
 
+    static class FoldNull extends OptimizerExpressionRule {
+
+        FoldNull() {
+            super(TransformDirection.UP);
+        }
+
+        @Override
+        protected Expression rule(Expression e) {
+            if (e instanceof IsNotNull) {
+                if (((IsNotNull) e).field().nullable() == false) {
+                    return new Literal(e.location(), Expressions.name(e), Boolean.TRUE, DataType.BOOLEAN);
+                }
+            }
+            // see https://github.com/elastic/elasticsearch/issues/34876
+            // similar for IsNull once it gets introduced
+
+            if (e instanceof In) {
+                In in = (In) e;
+                if (Expressions.isNull(in.value())) {
+                    return Literal.of(in, null);
+                }
+            }
+
+            if (e.nullable() && Expressions.anyMatch(e.children(), Expressions::isNull)) {
+                return Literal.of(e, null);
+            }
+
+            return e;
+        }
+    }
+
     static class ConstantFolding extends OptimizerExpressionRule {
 
         ConstantFolding() {
@@ -1128,6 +1165,38 @@ protected Expression rule(Expression e) {
             return e.foldable() ? Literal.of(e) : e;
         }
     }
+    
+    static class SimplifyCoalesce extends OptimizerExpressionRule {
+
+        SimplifyCoalesce() {
+            super(TransformDirection.DOWN);
+        }
+
+        @Override
+        protected Expression rule(Expression e) {
+            if (e instanceof Coalesce) {
+                Coalesce c = (Coalesce) e;
+
+                // find the first non-null foldable child (if any) and remove the rest
+                // while at it, exclude any nulls found
+                List<Expression> newChildren = new ArrayList<>();
+
+                for (Expression child : c.children()) {
+                    if (Expressions.isNull(child) == false) {
+                        newChildren.add(child);
+                        if (child.foldable()) {
+                            break;
+                        }
+                    }
+                }
+
+                if (newChildren.size() < c.children().size()) {
+                    return new Coalesce(c.location(), newChildren);
+                }
+            }
+            return e;
+        }
+    }
 
     static class BooleanSimplification extends OptimizerExpressionRule {
 
@@ -1275,7 +1344,7 @@ private Expression simplify(BinaryComparison bc) {
             }
 
             // false for equality
-            if (bc instanceof GreaterThan || bc instanceof LessThan) {
+            if (bc instanceof NotEquals || bc instanceof GreaterThan || bc instanceof LessThan) {
                 if (!l.nullable() && !r.nullable() && l.semanticEquals(r)) {
                     return FALSE;
                 }
@@ -1892,4 +1961,4 @@ protected LogicalPlan rule(LogicalPlan plan) {
     enum TransformDirection {
         UP, DOWN
     };
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
index 84bb213fb6805..40c9544bfd0c7 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
@@ -24,8 +24,7 @@
 import org.elasticsearch.xpack.sql.expression.function.Function;
 import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.sql.expression.function.scalar.Cast;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
-import org.elasticsearch.xpack.sql.expression.predicate.IsNotNull;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.expression.predicate.Range;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQueryPredicate;
@@ -33,6 +32,7 @@
 import org.elasticsearch.xpack.sql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Div;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mod;
@@ -44,6 +44,7 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.RLike;
@@ -165,7 +166,7 @@ public Expression visitComparison(ComparisonContext ctx) {
             case SqlBaseParser.EQ:
                 return new Equals(loc, left, right);
             case SqlBaseParser.NEQ:
-                return new Not(loc, new Equals(loc, left, right));
+                return new NotEquals(loc, left, right);
             case SqlBaseParser.LT:
                 return new LessThan(loc, left, right);
             case SqlBaseParser.LTE:
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java
index 2aee552907bdc..fcc0d50bd0515 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java
@@ -40,6 +40,7 @@
 import java.util.function.Function;
 
 import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.parser.AbstractBuilder.source;
 
 public class SqlParser {
 
@@ -215,7 +216,7 @@ public void exitNonReserved(SqlBaseParser.NonReservedContext context) {
      */
     private class CircuitBreakerListener extends SqlBaseBaseListener {
 
-        private static final short MAX_RULE_DEPTH = 100;
+        private static final short MAX_RULE_DEPTH = 200;
 
         // Keep current depth for every rule visited.
         // The totalDepth alone cannot be used as expressions like: e1 OR e2 OR e3 OR ...
@@ -225,16 +226,24 @@ private class CircuitBreakerListener extends SqlBaseBaseListener {
 
         @Override
         public void enterEveryRule(ParserRuleContext ctx) {
-            short currentDepth = depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 1, (short) 1);
-            if (currentDepth > MAX_RULE_DEPTH) {
-                throw new ParsingException("expression is too large to parse, (tree's depth exceeds {})", MAX_RULE_DEPTH);
+            if (ctx.getClass() != SqlBaseParser.UnquoteIdentifierContext.class &&
+                ctx.getClass() != SqlBaseParser.QuoteIdentifierContext.class &&
+                ctx.getClass() != SqlBaseParser.BackQuotedIdentifierContext.class) {
+                int currentDepth = depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 1, (short) 1);
+                if (currentDepth > MAX_RULE_DEPTH) {
+                    throw new ParsingException(source(ctx), "SQL statement too large; " +
+                        "halt parsing to prevent memory errors (stopped at depth {})", MAX_RULE_DEPTH);
+                }
             }
             super.enterEveryRule(ctx);
         }
 
         @Override
         public void exitEveryRule(ParserRuleContext ctx) {
-            depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 0, (short) -1);
+            // Avoid having negative numbers
+            if (depthCounts.containsKey(ctx.getClass().getSimpleName())) {
+                depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 0, (short) -1);
+            }
             super.exitEveryRule(ctx);
         }
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
index 3605898210fc5..c17c1311cccc6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
@@ -30,7 +30,7 @@
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.UnaryPipe;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.plan.physical.AggregateExec;
 import org.elasticsearch.xpack.sql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.sql.plan.physical.FilterExec;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java
index 9fcd542ef631d..21b6f20001bf7 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java
@@ -31,8 +31,6 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeHistogramFunction;
 import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
-import org.elasticsearch.xpack.sql.expression.predicate.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.Range;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.MultiMatchQueryPredicate;
@@ -40,12 +38,15 @@
 import org.elasticsearch.xpack.sql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.RLike;
@@ -536,16 +537,15 @@ protected QueryTranslation asQuery(BinaryComparison bc, boolean onAggs) {
                 //
                 // Agg context means HAVING -> PipelineAggs
                 //
-                ScriptTemplate script = bc.asScript();
                 if (onAggs) {
-                    aggFilter = new AggFilter(at.id().toString(), script);
+                    aggFilter = new AggFilter(at.id().toString(), bc.asScript());
                 }
                 else {
                     // query directly on the field
                     if (at instanceof FieldAttribute) {
                         query = wrapIfNested(translateQuery(bc), ne);
                     } else {
-                        query = new ScriptQuery(at.location(), script);
+                        query = new ScriptQuery(at.location(), bc.asScript());
                     }
                 }
                 return new QueryTranslation(query, aggFilter);
@@ -576,7 +576,7 @@ private static Query translateQuery(BinaryComparison bc) {
             if (bc instanceof LessThanOrEqual) {
                 return new RangeQuery(loc, name, null, false, value, true, format);
             }
-            if (bc instanceof Equals) {
+            if (bc instanceof Equals || bc instanceof NotEquals) {
                 if (bc.left() instanceof FieldAttribute) {
                     FieldAttribute fa = (FieldAttribute) bc.left();
                     // equality should always be against an exact match
@@ -585,7 +585,11 @@ private static Query translateQuery(BinaryComparison bc) {
                         name = fa.exactAttribute().name();
                     }
                 }
-                return new TermQuery(loc, name, value);
+                Query query = new TermQuery(loc, name, value);
+                if (bc instanceof NotEquals) {
+                    query = new NotQuery(loc, query);
+            }
+                return query;
             }
 
             throw new SqlIllegalArgumentException("Don't know how to translate binary comparison [{}] in [{}]", bc.right().nodeString(),
@@ -655,11 +659,10 @@ protected QueryTranslation asQuery(Range r, boolean onAggs) {
                 //
                 // Agg context means HAVING -> PipelineAggs
                 //
-                ScriptTemplate script = r.asScript();
                 Attribute at = ((NamedExpression) e).toAttribute();
 
                 if (onAggs) {
-                    aggFilter = new AggFilter(at.id().toString(), script);
+                    aggFilter = new AggFilter(at.id().toString(), r.asScript());
                 } else {
                     // typical range; no scripting involved
                     if (at instanceof FieldAttribute) {
@@ -669,7 +672,7 @@ protected QueryTranslation asQuery(Range r, boolean onAggs) {
                     }
                     // scripted query
                     else {
-                        query = new ScriptQuery(at.location(), script);
+                        query = new ScriptQuery(at.location(), r.asScript());
                     }
                 }
                 return new QueryTranslation(query, aggFilter);
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/AggFilter.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/AggFilter.java
index ea71156b0b05f..b22d2ed60d663 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/AggFilter.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/AggFilter.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.sql.expression.gen.script.Scripts;
 import org.elasticsearch.xpack.sql.util.Check;
 
 import java.util.Collection;
@@ -26,7 +27,8 @@ public class AggFilter extends PipelineAgg {
     public AggFilter(String name, ScriptTemplate scriptTemplate) {
         super(BUCKET_SELECTOR_ID_PREFIX + name);
         Check.isTrue(scriptTemplate != null, "a valid script is required");
-        this.scriptTemplate = scriptTemplate;
+        // make script null safe
+        this.scriptTemplate = Scripts.nullSafeFilter(scriptTemplate);
         this.aggPaths = scriptTemplate.aggPaths();
     }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/NotQuery.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/NotQuery.java
index b3d50b8149a43..1e76cb296fed7 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/NotQuery.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/NotQuery.java
@@ -9,10 +9,10 @@
 import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.xpack.sql.tree.Location;
 
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-
 import java.util.Objects;
 
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+
 public class NotQuery extends Query {
     private final Query child;
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/TermsQuery.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/TermsQuery.java
index 91ea49a8a3ce3..66d206f829a32 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/TermsQuery.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/TermsQuery.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.xpack.sql.expression.Expression;
 import org.elasticsearch.xpack.sql.expression.Foldables;
 import org.elasticsearch.xpack.sql.tree.Location;
-import org.elasticsearch.xpack.sql.type.DataType;
+import org.elasticsearch.xpack.sql.type.DataTypes;
 
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -27,7 +27,7 @@ public class TermsQuery extends LeafQuery {
     public TermsQuery(Location location, String term, List<Expression> values) {
         super(location);
         this.term = term;
-        values.removeIf(e -> e.dataType() == DataType.NULL);
+        values.removeIf(e -> DataTypes.isNull(e.dataType()));
         if (values.isEmpty()) {
             this.values = Collections.emptySet();
         } else {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/Rule.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/Rule.java
index d63357e151aed..5bdd74fefc3df 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/Rule.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/Rule.java
@@ -7,8 +7,8 @@
 
 import java.util.function.UnaryOperator;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.xpack.sql.tree.Node;
 import org.elasticsearch.xpack.sql.util.ReflectionUtils;
 
@@ -22,7 +22,7 @@
  */
 public abstract class Rule<E extends T, T extends Node<T>> implements UnaryOperator<T> {
 
-    protected Logger log = Loggers.getLogger(getClass());
+    protected Logger log = LogManager.getLogger(getClass());
 
     private final String name;
     private final Class<E> typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass());
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/RuleExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/RuleExecutor.java
index 2936e6342add9..90f3537c62561 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/RuleExecutor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/rule/RuleExecutor.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.rule;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.sql.tree.Node;
 import org.elasticsearch.xpack.sql.tree.NodeUtils;
@@ -18,7 +18,7 @@
 
 public abstract class RuleExecutor<TreeType extends Node<TreeType>> {
 
-    private final Logger log = Loggers.getLogger(getClass());
+    private final Logger log = LogManager.getLogger(getClass());
 
     public static class Limiter {
         public static final Limiter DEFAULT = new Limiter(100);
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java
index 3312f449ec622..26436c614f565 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java
@@ -102,7 +102,10 @@ public static Conversion conversionFor(DataType from, DataType to) {
         if (from == to) {
             return Conversion.IDENTITY;
         }
-        if (to == DataType.NULL) {
+        if (to == DataType.NULL || from == DataType.NULL) {
+            return Conversion.NULL;
+        }
+        if (from == DataType.NULL) {
             return Conversion.NULL;
         }
         
diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
index 998dab84783f0..dd8a8ad800a1e 100644
--- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
+++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
@@ -20,10 +20,12 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
 # Comparison
 #
   Boolean eq(Object, Object)
+  Boolean neq(Object, Object)
   Boolean lt(Object, Object)
   Boolean lte(Object, Object)
   Boolean gt(Object, Object)
   Boolean gte(Object, Object)
+  Boolean in(Object, java.util.List)
 
 #
 # Logical
@@ -33,6 +35,11 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
   Boolean not(Boolean)
   Boolean notNull(Object)    
 
+#
+# Null
+#
+  Object coalesce(java.util.List)
+
 #
 # Regex
 #
@@ -107,4 +114,4 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
   String  space(Number)
   String  substring(String, Number, Number)
   String  ucase(String)
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
index 512b2da732a77..3b03079ca723b 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
@@ -49,6 +49,38 @@ public void testMissingIndex() {
     public void testMissingColumn() {
         assertEquals("1:8: Unknown column [xxx]", verify("SELECT xxx FROM test"));
     }
+    
+    public void testMissingColumnWithWildcard() {
+        assertEquals("1:8: Unknown column [xxx]", verify("SELECT xxx.* FROM test"));
+    }
+    
+    public void testMisspelledColumnWithWildcard() {
+        assertEquals("1:8: Unknown column [tex], did you mean [text]?", verify("SELECT tex.* FROM test"));
+    }
+    
+    public void testColumnWithNoSubFields() {
+        assertEquals("1:8: Cannot determine columns for [text.*]", verify("SELECT text.* FROM test"));
+    }
+    
+    public void testMultipleColumnsWithWildcard1() {
+        assertEquals("1:14: Unknown column [a]\n" + 
+                "line 1:17: Unknown column [b]\n" + 
+                "line 1:22: Unknown column [c]\n" + 
+                "line 1:25: Unknown column [tex], did you mean [text]?", verify("SELECT bool, a, b.*, c, tex.* FROM test"));
+    }
+    
+    public void testMultipleColumnsWithWildcard2() {
+        assertEquals("1:8: Unknown column [tex], did you mean [text]?\n" + 
+                "line 1:21: Unknown column [a]\n" + 
+                "line 1:24: Unknown column [dat], did you mean [date]?\n" + 
+                "line 1:31: Unknown column [c]", verify("SELECT tex.*, bool, a, dat.*, c FROM test"));
+    }
+    
+    public void testMultipleColumnsWithWildcard3() {
+        assertEquals("1:8: Unknown column [ate], did you mean [date]?\n" + 
+                "line 1:21: Unknown column [keyw], did you mean [keyword]?\n" + 
+                "line 1:29: Unknown column [da], did you mean [date]?" , verify("SELECT ate.*, bool, keyw.*, da FROM test"));
+    }
 
     public void testMisspelledColumn() {
         assertEquals("1:8: Unknown column [txt], did you mean [text]?", verify("SELECT txt FROM test"));
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ProcessorTests.java
new file mode 100644
index 0000000000000..e3d51f1f7dd87
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ProcessorTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.expression;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.sql.tree.NodeSubclassTests;
+import org.junit.BeforeClass;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+
+import static java.util.stream.Collectors.toCollection;
+
+
+public class ProcessorTests extends ESTestCase {
+
+    private static List<Class<? extends Processor>> processors;
+
+    @BeforeClass
+    public static void init() throws Exception {
+        processors = NodeSubclassTests.subclassesOf(Processor.class);
+    }
+
+
+    public void testProcessorRegistration() throws Exception {
+        LinkedHashSet<String> registered = Processors.getNamedWriteables().stream()
+                .map(e -> e.name)
+                .collect(toCollection(LinkedHashSet::new));
+
+        // discover available processors
+        int missing = processors.size() - registered.size();
+
+
+        if (missing > 0) {
+            List<String> notRegistered = new ArrayList<>();
+            for (Class<? extends Processor> proc : processors) {
+                String procName = proc.getName();
+                assertTrue(procName + " does NOT implement NamedWriteable", NamedWriteable.class.isAssignableFrom(proc));
+                Field name = null;
+                String value = null;
+                try {
+                    name = proc.getField("NAME");
+                } catch (Exception ex) {
+                    fail(procName + " does NOT provide a NAME field\n" + ex);
+                }
+                try {
+                    value = name.get(proc).toString();
+                } catch (Exception ex) {
+                    fail(procName + " does NOT provide a static NAME field\n" + ex);
+                }
+                if (!registered.contains(value)) {
+                    notRegistered.add(procName);
+                }
+            }
+            
+            fail(missing + " processor(s) not registered : " + notRegistered);
+        } else {
+            assertEquals("Detection failed: discovered more registered processors than classes", 0, missing);
+        }
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessorTests.java
new file mode 100644
index 0000000000000..ec29e912a2c90
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/logical/BinaryLogicProcessorTests.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.expression.predicate.logical;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
+import org.elasticsearch.xpack.sql.expression.gen.processor.ConstantProcessor;
+import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
+
+public class BinaryLogicProcessorTests extends AbstractWireSerializingTestCase<BinaryLogicProcessor> {
+
+    private static final Processor FALSE = new ConstantProcessor(false);
+    private static final Processor TRUE = new ConstantProcessor(true);
+    private static final Processor NULL = new ConstantProcessor((Object) null);
+
+    public static BinaryLogicProcessor randomProcessor() {
+        return new BinaryLogicProcessor(
+                new ConstantProcessor(randomFrom(Boolean.FALSE, Boolean.TRUE, null)),
+                new ConstantProcessor(randomFrom(Boolean.FALSE, Boolean.TRUE, null)),
+                randomFrom(BinaryLogicProcessor.BinaryLogicOperation.values()));
+    }
+
+    @Override
+    protected BinaryLogicProcessor createTestInstance() {
+        return randomProcessor();
+    }
+
+    @Override
+    protected Reader<BinaryLogicProcessor> instanceReader() {
+        return BinaryLogicProcessor::new;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Processors.getNamedWriteables());
+    }
+
+    public void testOR() {
+        assertEquals(true, new BinaryLogicProcessor(TRUE, FALSE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertEquals(true, new BinaryLogicProcessor(FALSE, TRUE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertEquals(false, new BinaryLogicProcessor(FALSE, FALSE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertEquals(true, new BinaryLogicProcessor(TRUE, TRUE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+    }
+
+    public void testORNullHandling() {
+        assertEquals(true, new BinaryLogicProcessor(TRUE, NULL, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertEquals(true, new BinaryLogicProcessor(NULL, TRUE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertNull(new BinaryLogicProcessor(FALSE, NULL, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertNull(new BinaryLogicProcessor(NULL, FALSE, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+        assertNull(new BinaryLogicProcessor(NULL, NULL, BinaryLogicProcessor.BinaryLogicOperation.OR).process(null));
+    }
+
+    public void testAnd() {
+        assertEquals(false, new BinaryLogicProcessor(TRUE, FALSE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertEquals(false, new BinaryLogicProcessor(FALSE, TRUE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertEquals(false, new BinaryLogicProcessor(FALSE, FALSE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertEquals(true, new BinaryLogicProcessor(TRUE, TRUE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+    }
+
+    public void testAndNullHandling() {
+        assertNull(new BinaryLogicProcessor(TRUE, NULL, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertNull(new BinaryLogicProcessor(NULL, TRUE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertEquals(false, new BinaryLogicProcessor(FALSE, NULL, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertEquals(false, new BinaryLogicProcessor(NULL, FALSE, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+        assertNull(new BinaryLogicProcessor(NULL, NULL, BinaryLogicProcessor.BinaryLogicOperation.AND).process(null));
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
index 0761ec5f2fa21..394818be2408a 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
@@ -11,12 +11,6 @@
 import org.elasticsearch.xpack.sql.expression.Literal;
 import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
 import org.elasticsearch.xpack.sql.expression.gen.processor.ConstantProcessor;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.BinaryComparisonProcessor;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Equals;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThan;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThanOrEqual;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThan;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThanOrEqual;
 
 import static org.elasticsearch.xpack.sql.tree.Location.EMPTY;
 
@@ -48,6 +42,11 @@ public void testEq() {
         assertEquals(false, new Equals(EMPTY, l(3), l(4)).makePipe().asProcessor().process(null));
     }
 
+    public void testNEq() {
+        assertEquals(false, new NotEquals(EMPTY, l(4), l(4)).makePipe().asProcessor().process(null));
+        assertEquals(true, new NotEquals(EMPTY, l(3), l(4)).makePipe().asProcessor().process(null));
+    }
+
     public void testGt() {
         assertEquals(true, new GreaterThan(EMPTY, l(4), l(3)).makePipe().asProcessor().process(null));
         assertEquals(false, new GreaterThan(EMPTY, l(3), l(4)).makePipe().asProcessor().process(null));
@@ -73,14 +72,15 @@ public void testLte() {
     }
     
     public void testHandleNull() {
-        assertNull(new Equals(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null));
-        assertNull(new GreaterThan(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null));
-        assertNull(new GreaterThanOrEqual(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null));
-        assertNull(new LessThan(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null));
-        assertNull(new LessThanOrEqual(EMPTY, l(null), l(3)).makePipe().asProcessor().process(null));
+        assertNull(new Equals(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
+        assertNull(new NotEquals(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
+        assertNull(new GreaterThan(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
+        assertNull(new GreaterThanOrEqual(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
+        assertNull(new LessThan(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
+        assertNull(new LessThanOrEqual(EMPTY, Literal.NULL, l(3)).makePipe().asProcessor().process(null));
     }
     
     private static Literal l(Object value) {
         return Literal.of(EMPTY, value);
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessorTests.java
similarity index 94%
rename from x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InProcessorTests.java
rename to x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessorTests.java
index 59c9df1f89996..3303072e50078 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InProcessorTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.sql.expression.predicate;
+package org.elasticsearch.xpack.sql.expression.predicate.operator.comparison;
 
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.sql.expression.Literal;
 import org.elasticsearch.xpack.sql.expression.function.scalar.Processors;
 import org.elasticsearch.xpack.sql.expression.gen.processor.ConstantProcessor;
-import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InProcessor;
 
 import java.util.Arrays;
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InTests.java
similarity index 95%
rename from x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InTests.java
rename to x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InTests.java
index 984fb833feb08..c78014afbc471 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/InTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/predicate/operator/comparison/InTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.sql.expression.predicate;
+package org.elasticsearch.xpack.sql.expression.predicate.operator.comparison;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.expression.Literal;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
index 0246499f7f9c0..d63f588cf2281 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.sql.expression.function.aggregate.AggregateFunction;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Count;
 import org.elasticsearch.xpack.sql.expression.function.scalar.Cast;
+import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayName;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfMonth;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfYear;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.MonthOfYear;
@@ -28,15 +29,18 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.ASin;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.ATan;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.Abs;
+import org.elasticsearch.xpack.sql.expression.function.scalar.math.Cos;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.E;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.Floor;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Ascii;
+import org.elasticsearch.xpack.sql.expression.function.scalar.string.Repeat;
 import org.elasticsearch.xpack.sql.expression.predicate.BinaryOperator;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
-import org.elasticsearch.xpack.sql.expression.predicate.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.Range;
+import org.elasticsearch.xpack.sql.expression.predicate.conditional.Coalesce;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.sql.expression.predicate.logical.Or;
+import org.elasticsearch.xpack.sql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Div;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mod;
@@ -45,8 +49,10 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.GreaterThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.LessThanOrEqual;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.RLike;
@@ -56,10 +62,12 @@
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.CombineBinaryComparisons;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.CombineProjections;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.ConstantFolding;
+import org.elasticsearch.xpack.sql.optimizer.Optimizer.FoldNull;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.PropagateEquals;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.PruneDuplicateFunctions;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.PruneSubqueryAliases;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer.ReplaceFoldableAttributes;
+import org.elasticsearch.xpack.sql.optimizer.Optimizer.SimplifyCoalesce;
 import org.elasticsearch.xpack.sql.plan.logical.Filter;
 import org.elasticsearch.xpack.sql.plan.logical.LocalRelation;
 import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan;
@@ -72,6 +80,7 @@
 import org.elasticsearch.xpack.sql.tree.NodeInfo;
 import org.elasticsearch.xpack.sql.type.DataType;
 import org.elasticsearch.xpack.sql.type.EsField;
+import org.elasticsearch.xpack.sql.util.CollectionUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -260,6 +269,7 @@ public void testConstantFoldingBinaryComparison() {
         assertEquals(Literal.FALSE, new ConstantFolding().rule(new GreaterThan(EMPTY, TWO, THREE)).canonical());
         assertEquals(Literal.FALSE, new ConstantFolding().rule(new GreaterThanOrEqual(EMPTY, TWO, THREE)).canonical());
         assertEquals(Literal.FALSE, new ConstantFolding().rule(new Equals(EMPTY, TWO, THREE)).canonical());
+        assertEquals(Literal.TRUE, new ConstantFolding().rule(new NotEquals(EMPTY, TWO, THREE)).canonical());
         assertEquals(Literal.TRUE, new ConstantFolding().rule(new LessThanOrEqual(EMPTY, TWO, THREE)).canonical());
         assertEquals(Literal.TRUE, new ConstantFolding().rule(new LessThan(EMPTY, TWO, THREE)).canonical());
     }
@@ -340,7 +350,7 @@ public void testConstantFoldingIn_LeftValueNotFoldable() {
     public void testConstantFoldingIn_RightValueIsNull() {
         In in = new In(EMPTY, getFieldAttribute(), Arrays.asList(NULL, NULL));
         Literal result= (Literal) new ConstantFolding().rule(in);
-        assertEquals(false, result.value());
+        assertNull(result.value());
     }
 
     public void testConstantFoldingIn_LeftValueIsNull() {
@@ -374,12 +384,73 @@ private static Object foldOperator(BinaryOperator<?, ?, ?, ?> b) {
         return ((Literal) new ConstantFolding().rule(b)).value();
     }
 
+    public void testNullFoldingIsNotNull() {
+        assertEquals(Literal.TRUE, new FoldNull().rule(new IsNotNull(EMPTY, Literal.TRUE)));
+    }
+
+    public void testGenericNullableExpression() {
+        FoldNull rule = new FoldNull();
+        // date-time
+        assertNullLiteral(rule.rule(new DayName(EMPTY, Literal.NULL, randomTimeZone())));
+        // math function
+        assertNullLiteral(rule.rule(new Cos(EMPTY, Literal.NULL)));
+        // string function
+        assertNullLiteral(rule.rule(new Ascii(EMPTY, Literal.NULL)));
+        assertNullLiteral(rule.rule(new Repeat(EMPTY, getFieldAttribute(), Literal.NULL)));
+        // arithmetic
+        assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute(), Literal.NULL)));
+        // comparison
+        assertNullLiteral(rule.rule(new GreaterThan(EMPTY, getFieldAttribute(), Literal.NULL)));
+        // regex
+        assertNullLiteral(rule.rule(new RLike(EMPTY, getFieldAttribute(), Literal.NULL)));
+    }
+
+    public void testSimplifyCoalesceNulls() {
+        Expression e = new SimplifyCoalesce().rule(new Coalesce(EMPTY, asList(Literal.NULL, Literal.NULL)));
+        assertEquals(Coalesce.class, e.getClass());
+        assertEquals(0, e.children().size());
+    }
+
+    public void testSimplifyCoalesceRandomNulls() {
+        Expression e = new SimplifyCoalesce().rule(new Coalesce(EMPTY, randomListOfNulls()));
+        assertEquals(Coalesce.class, e.getClass());
+        assertEquals(0, e.children().size());
+    }
+
+    public void testSimplifyCoalesceRandomNullsWithValue() {
+        Expression e = new SimplifyCoalesce().rule(new Coalesce(EMPTY,
+                CollectionUtils.combine(
+                        CollectionUtils.combine(randomListOfNulls(), Literal.TRUE, Literal.FALSE, Literal.TRUE),
+                        randomListOfNulls())));
+        assertEquals(1, e.children().size());
+        assertEquals(Literal.TRUE, e.children().get(0));
+    }
+
+    private List<Expression> randomListOfNulls() {
+        return asList(randomArray(1, 10, i -> new Literal[i], () -> Literal.NULL));
+    }
+
+    public void testSimplifyCoalesceFirstLiteral() {
+        Expression e = new SimplifyCoalesce()
+                .rule(new Coalesce(EMPTY,
+                        Arrays.asList(Literal.NULL, Literal.TRUE, Literal.FALSE, new Abs(EMPTY, getFieldAttribute()))));
+        assertEquals(Coalesce.class, e.getClass());
+        assertEquals(1, e.children().size());
+        assertEquals(Literal.TRUE, e.children().get(0));
+    }
+
     //
     // Logical simplifications
     //
 
+    private void assertNullLiteral(Expression expression) {
+        assertEquals(Literal.class, expression.getClass());
+        assertNull(expression.fold());
+    }
+
     public void testBinaryComparisonSimplification() {
         assertEquals(Literal.TRUE, new BinaryComparisonSimplification().rule(new Equals(EMPTY, FIVE, FIVE)));
+        assertEquals(Literal.FALSE, new BinaryComparisonSimplification().rule(new NotEquals(EMPTY, FIVE, FIVE)));
         assertEquals(Literal.TRUE, new BinaryComparisonSimplification().rule(new GreaterThanOrEqual(EMPTY, FIVE, FIVE)));
         assertEquals(Literal.TRUE, new BinaryComparisonSimplification().rule(new LessThanOrEqual(EMPTY, FIVE, FIVE)));
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
index 0ee0c9bcca122..70ff5ac8c0e7d 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Neg;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.Sub;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.sql.type.DataType;
 
 import static org.hamcrest.core.StringStartsWith.startsWith;
@@ -158,6 +160,22 @@ public void testComplexArithmetic() {
         assertEquals("2", ((Literal) sub2.children().get(1)).name());
     }
 
+    public void testEquals() {
+        Expression expr = parser.createExpression("a = 10");
+        assertEquals(Equals.class, expr.getClass());
+        Equals eq = (Equals) expr;
+        assertEquals("(a) == 10", eq.name());
+        assertEquals(2, eq.children().size());
+    }
+
+    public void testNotEquals() {
+        Expression expr = parser.createExpression("a != 10");
+        assertEquals(NotEquals.class, expr.getClass());
+        NotEquals neq = (NotEquals) expr;
+        assertEquals("(a) != 10", neq.name());
+        assertEquals(2, neq.children().size());
+    }
+
     public void testCastWithUnquotedDataType() {
         Expression expr = parser.createExpression("CAST(10*2 AS long)");
         assertEquals(Cast.class, expr.getClass());
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
index aab25349a1d0e..2794481dc8075 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java
@@ -138,71 +138,104 @@ public void testMultiMatchQuery() {
         assertThat(mmqp.optionMap(), hasEntry("fuzzy_rewrite", "scoring_boolean"));
     }
 
+    public void testLimitToPreventStackOverflowFromLongListOfQuotedIdentifiers() {
+        // Create expression in the form of "t"."field","t"."field", ...
+
+        // 200 elements is ok
+        new SqlParser().createStatement("SELECT " +
+            Joiner.on(",").join(nCopies(200, "\"t\".\"field\"")) + " FROM t");
+
+        // 201 elements parser's "circuit breaker" is triggered
+        ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createStatement("SELECT " +
+            Joiner.on(",").join(nCopies(201, "\"t\".\"field\"")) + " FROM t"));
+        assertEquals("line 1:2409: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
+    }
+
+    public void testLimitToPreventStackOverflowFromLongListOfUnQuotedIdentifiers() {
+        // Create expression in the form of t.field,t.field, ...
+
+        // 250 elements is ok
+        new SqlParser().createStatement("SELECT " +
+            Joiner.on(",").join(nCopies(200, "t.field")) + " FROM t");
+
+        // 251 elements parser's "circuit breaker" is triggered
+        ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createStatement("SELECT " +
+            Joiner.on(",").join(nCopies(201, "t.field")) + " FROM t"));
+        assertEquals("line 1:1609: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
+    }
+
     public void testLimitToPreventStackOverflowFromLargeUnaryBooleanExpression() {
         // Create expression in the form of NOT(NOT(NOT ... (b) ...)
 
-        // 40 elements is ok
+        // 99 elements is ok
         new SqlParser().createExpression(
-            Joiner.on("").join(nCopies(40, "NOT(")).concat("b").concat(Joiner.on("").join(nCopies(40, ")"))));
+            Joiner.on("").join(nCopies(99, "NOT(")).concat("b").concat(Joiner.on("").join(nCopies(99, ")"))));
 
         // 100 elements parser's "circuit breaker" is triggered
         ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createExpression(
             Joiner.on("").join(nCopies(100, "NOT(")).concat("b").concat(Joiner.on("").join(nCopies(100, ")")))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+        assertEquals("line 1:402: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     public void testLimitToPreventStackOverflowFromLargeBinaryBooleanExpression() {
         // Create expression in the form of a = b OR a = b OR ... a = b
 
-        // 50 elements is ok
-        new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(50, "a = b")));
+        // 100 elements is ok
+        new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(100, "a = b")));
 
-        // 100 elements parser's "circuit breaker" is triggered
+        // 101 elements parser's "circuit breaker" is triggered
         ParsingException e = expectThrows(ParsingException.class, () ->
-            new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(100, "a = b"))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+            new SqlParser().createExpression(Joiner.on(" OR ").join(nCopies(101, "a = b"))));
+        assertEquals("line 1:902: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     public void testLimitToPreventStackOverflowFromLargeUnaryArithmeticExpression() {
         // Create expression in the form of abs(abs(abs ... (i) ...)
 
-        // 50 elements is ok
+        // 199 elements is ok
         new SqlParser().createExpression(
-            Joiner.on("").join(nCopies(50, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(50, ")"))));
+            Joiner.on("").join(nCopies(199, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(199, ")"))));
 
-        // 101 elements parser's "circuit breaker" is triggered
+        // 200 elements parser's "circuit breaker" is triggered
         ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createExpression(
-            Joiner.on("").join(nCopies(101, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(101, ")")))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+            Joiner.on("").join(nCopies(200, "abs(")).concat("i").concat(Joiner.on("").join(nCopies(200, ")")))));
+        assertEquals("line 1:802: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     public void testLimitToPreventStackOverflowFromLargeBinaryArithmeticExpression() {
         // Create expression in the form of a + a + a + ... + a
 
-        // 100 elements is ok
-        new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(100, "a")));
+        // 200 elements is ok
+        new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(200, "a")));
 
-        // 101 elements parser's "circuit breaker" is triggered
+        // 201 elements parser's "circuit breaker" is triggered
         ParsingException e = expectThrows(ParsingException.class, () ->
-            new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(101, "a"))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+            new SqlParser().createExpression(Joiner.on(" + ").join(nCopies(201, "a"))));
+        assertEquals("line 1:802: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     public void testLimitToPreventStackOverflowFromLargeSubselectTree() {
         // Test with queries in the form of `SELECT * FROM (SELECT * FROM (... t) ...)
 
-        // 100 elements is ok
+        // 200 elements is ok
         new SqlParser().createStatement(
-            Joiner.on(" (").join(nCopies(100, "SELECT * FROM"))
+            Joiner.on(" (").join(nCopies(200, "SELECT * FROM"))
                 .concat("t")
-                .concat(Joiner.on("").join(nCopies(99, ")"))));
+                .concat(Joiner.on("").join(nCopies(199, ")"))));
 
-        // 101 elements parser's "circuit breaker" is triggered
+        // 201 elements parser's "circuit breaker" is triggered
         ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createStatement(
-            Joiner.on(" (").join(nCopies(101, "SELECT * FROM"))
+            Joiner.on(" (").join(nCopies(201, "SELECT * FROM"))
                 .concat("t")
-                .concat(Joiner.on("").join(nCopies(100, ")")))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+                .concat(Joiner.on("").join(nCopies(200, ")")))));
+        assertEquals("line 1:3002: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     public void testLimitToPreventStackOverflowFromLargeComplexSubselectTree() {
@@ -210,14 +243,15 @@ public void testLimitToPreventStackOverflowFromLargeComplexSubselectTree() {
 
         new SqlParser().createStatement(
             Joiner.on(" (").join(nCopies(20, "SELECT ")).
-                concat(Joiner.on(" OR ").join(nCopies(50, "true"))).concat(" FROM")
+                concat(Joiner.on(" OR ").join(nCopies(180, "true"))).concat(" FROM")
                 .concat("t").concat(Joiner.on("").join(nCopies(19, ")"))));
 
         ParsingException e = expectThrows(ParsingException.class, () -> new SqlParser().createStatement(
             Joiner.on(" (").join(nCopies(20, "SELECT ")).
-                concat(Joiner.on(" OR ").join(nCopies(100, "true"))).concat(" FROM")
+                concat(Joiner.on(" OR ").join(nCopies(190, "true"))).concat(" FROM")
                 .concat("t").concat(Joiner.on("").join(nCopies(19, ")")))));
-        assertEquals("expression is too large to parse, (tree's depth exceeds 100)", e.getErrorMessage());
+        assertEquals("line 1:1628: SQL statement too large; halt parsing to prevent memory errors (stopped at depth 200)",
+            e.getMessage());
     }
 
     private LogicalPlan parseStatement(String sql) {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
index 5fac14e2397db..77606ab1390dd 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
@@ -5,13 +5,14 @@
  */
 package org.elasticsearch.xpack.sql.planner;
 
-import org.elasticsearch.test.AbstractBuilderTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
 import org.elasticsearch.xpack.sql.analysis.index.EsIndex;
 import org.elasticsearch.xpack.sql.analysis.index.IndexResolution;
 import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
+import org.elasticsearch.xpack.sql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.sql.plan.physical.LocalExec;
 import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.sql.session.EmptyExecutable;
@@ -25,7 +26,7 @@
 
 import static org.hamcrest.Matchers.startsWith;
 
-public class QueryFolderTests extends AbstractBuilderTestCase {
+public class QueryFolderTests extends ESTestCase {
 
     private static SqlParser parser;
     private static Analyzer analyzer;
@@ -64,6 +65,24 @@ public void testFoldingToLocalExecWithProject() {
         assertThat(ee.output().get(0).toString(), startsWith("keyword{f}#"));
     }
 
+    public void testFoldingOfIsNotNull() {
+        PhysicalPlan p = plan("SELECT keyword FROM test WHERE (keyword IS NULL) IS NOT NULL");
+        assertEquals(EsQueryExec.class, p.getClass());
+        EsQueryExec ee = (EsQueryExec) p;
+        assertEquals(1, ee.output().size());
+        assertThat(ee.output().get(0).toString(), startsWith("keyword{f}#"));
+    }
+
+    public void testFoldingToLocalExecWithNullFilter() {
+        PhysicalPlan p = plan("SELECT keyword FROM test WHERE null IN (1, 2)");
+        assertEquals(LocalExec.class, p.getClass());
+        LocalExec le = (LocalExec) p;
+        assertEquals(EmptyExecutable.class, le.executable().getClass());
+        EmptyExecutable ee = (EmptyExecutable) le.executable();
+        assertEquals(1, ee.output().size());
+        assertThat(ee.output().get(0).toString(), startsWith("keyword{f}#"));
+    }
+
     public void testFoldingToLocalExecWithProject_FoldableIn() {
         PhysicalPlan p = plan("SELECT keyword FROM test WHERE int IN (null, null)");
         assertEquals(LocalExec.class, p.getClass());
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index c1e5a0d2dafad..4423887c16135 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.planner;
 
-import org.elasticsearch.test.AbstractBuilderTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
 import org.elasticsearch.xpack.sql.analysis.index.EsIndex;
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.sql.plan.logical.Project;
 import org.elasticsearch.xpack.sql.planner.QueryTranslator.QueryTranslation;
+import org.elasticsearch.xpack.sql.querydsl.agg.AggFilter;
 import org.elasticsearch.xpack.sql.querydsl.query.Query;
 import org.elasticsearch.xpack.sql.querydsl.query.RangeQuery;
 import org.elasticsearch.xpack.sql.querydsl.query.ScriptQuery;
@@ -29,13 +30,13 @@
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.TimeZone;
 
-import static org.hamcrest.core.StringStartsWith.startsWith;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.startsWith;
 
-public class QueryTranslatorTests extends AbstractBuilderTestCase {
+public class QueryTranslatorTests extends ESTestCase {
 
     private static SqlParser parser;
     private static Analyzer analyzer;
@@ -160,7 +161,7 @@ public void testLikeConstructsNotSupported() {
         assertEquals("Scalar function (LTRIM(keyword)) not allowed (yet) as arguments for LIKE", ex.getMessage());
     }
 
-    public void testTranslateInExpression_WhereClause() throws IOException {
+    public void testTranslateInExpression_WhereClause() {
         LogicalPlan p = plan("SELECT * FROM test WHERE keyword IN ('foo', 'bar', 'lala', 'foo', concat('la', 'la'))");
         assertTrue(p instanceof Project);
         assertTrue(p.children().get(0) instanceof Filter);
@@ -170,10 +171,11 @@ public void testTranslateInExpression_WhereClause() throws IOException {
         Query query = translation.query;
         assertTrue(query instanceof TermsQuery);
         TermsQuery tq = (TermsQuery) query;
-        assertEquals("keyword:(bar foo lala)", tq.asBuilder().toQuery(createShardContext()).toString());
+        assertEquals("{\"terms\":{\"keyword\":[\"foo\",\"bar\",\"lala\"],\"boost\":1.0}}",
+            tq.asBuilder().toString().replaceAll("\\s", ""));
     }
 
-    public void testTranslateInExpression_WhereClauseAndNullHandling() throws IOException {
+    public void testTranslateInExpression_WhereClauseAndNullHandling() {
         LogicalPlan p = plan("SELECT * FROM test WHERE keyword IN ('foo', null, 'lala', null, 'foo', concat('la', 'la'))");
         assertTrue(p instanceof Project);
         assertTrue(p.children().get(0) instanceof Filter);
@@ -183,7 +185,8 @@ public void testTranslateInExpression_WhereClauseAndNullHandling() throws IOExce
         Query query = translation.query;
         assertTrue(query instanceof TermsQuery);
         TermsQuery tq = (TermsQuery) query;
-        assertEquals("keyword:(foo lala)", tq.asBuilder().toQuery(createShardContext()).toString());
+        assertEquals("{\"terms\":{\"keyword\":[\"foo\",\"lala\"],\"boost\":1.0}}",
+            tq.asBuilder().toString().replaceAll("\\s", ""));
     }
 
     public void testTranslateInExpressionInvalidValues_WhereClause() {
@@ -194,32 +197,68 @@ public void testTranslateInExpressionInvalidValues_WhereClause() {
         assertFalse(condition.foldable());
         SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
         assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; " +
-                "offender [keyword] in [keyword IN(foo, bar, keyword)]", ex.getMessage());
+                "offender [keyword] in [keyword IN (foo, bar, keyword)]", ex.getMessage());
     }
 
-    public void testTranslateInExpression_HavingClause_Painless() {
-        LogicalPlan p = plan("SELECT keyword, max(int) FROM test GROUP BY keyword HAVING max(int) in (10, 20, 30 - 10)");
+    public void testTranslateInExpression_WhereClause_Painless() {
+        LogicalPlan p = plan("SELECT int FROM test WHERE POWER(int, 2) IN (10, null, 20, 30 - 10)");
         assertTrue(p instanceof Project);
         assertTrue(p.children().get(0) instanceof Filter);
         Expression condition = ((Filter) p.children().get(0)).condition();
         assertFalse(condition.foldable());
         QueryTranslation translation = QueryTranslator.toQuery(condition, false);
+        assertNull(translation.aggFilter);
         assertTrue(translation.query instanceof ScriptQuery);
-        ScriptQuery sq = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(params.a0==10 || params.a0==20)", sq.script().toString());
-        assertThat(sq.script().params().toString(), startsWith("[{a=MAX(int){a->"));
+        ScriptQuery sc = (ScriptQuery) translation.query;
+        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(" +
+                "InternalSqlScriptUtils.power(InternalSqlScriptUtils.docValue(doc,params.v0),params.v1), params.v2))",
+            sc.script().toString());
+        assertEquals("[{v=int}, {v=2}, {v=[10.0, null, 20.0]}]", sc.script().params().toString());
     }
 
-    public void testTranslateInExpression_HavingClauseAndNullHandling_Painless() {
-        LogicalPlan p = plan("SELECT keyword, max(int) FROM test GROUP BY keyword HAVING max(int) in (10, null, 20, null, 30 - 10)");
+    public void testTranslateInExpression_HavingClause_Painless() {
+        LogicalPlan p = plan("SELECT keyword, max(int) FROM test GROUP BY keyword HAVING max(int) IN (10, 20, 30 - 10)");
         assertTrue(p instanceof Project);
         assertTrue(p.children().get(0) instanceof Filter);
         Expression condition = ((Filter) p.children().get(0)).condition();
         assertFalse(condition.foldable());
-        QueryTranslation translation = QueryTranslator.toQuery(condition, false);
-        assertTrue(translation.query instanceof ScriptQuery);
-        ScriptQuery sq = (ScriptQuery) translation.query;
-        assertEquals("InternalSqlScriptUtils.nullSafeFilter(params.a0==10 || params.a0==20)", sq.script().toString());
-        assertThat(sq.script().params().toString(), startsWith("[{a=MAX(int){a->"));
+        QueryTranslation translation = QueryTranslator.toQuery(condition, true);
+        assertNull(translation.query);
+        AggFilter aggFilter = translation.aggFilter;
+        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+            aggFilter.scriptTemplate().toString());
+        assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=MAX(int){a->"));
+        assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10, 20]}]"));
+    }
+
+    public void testTranslateInExpression_HavingClause_PainlessOneArg() {
+        LogicalPlan p = plan("SELECT keyword, max(int) FROM test GROUP BY keyword HAVING max(int) IN (10, 30 - 20)");
+        assertTrue(p instanceof Project);
+        assertTrue(p.children().get(0) instanceof Filter);
+        Expression condition = ((Filter) p.children().get(0)).condition();
+        assertFalse(condition.foldable());
+        QueryTranslation translation = QueryTranslator.toQuery(condition, true);
+        assertNull(translation.query);
+        AggFilter aggFilter = translation.aggFilter;
+        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+            aggFilter.scriptTemplate().toString());
+        assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=MAX(int){a->"));
+        assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10]}]"));
+
+    }
+
+    public void testTranslateInExpression_HavingClause_PainlessAndNullHandling() {
+        LogicalPlan p = plan("SELECT keyword, max(int) FROM test GROUP BY keyword HAVING max(int) IN (10, null, 20, 30, null, 30 - 10)");
+        assertTrue(p instanceof Project);
+        assertTrue(p.children().get(0) instanceof Filter);
+        Expression condition = ((Filter) p.children().get(0)).condition();
+        assertFalse(condition.foldable());
+        QueryTranslation translation = QueryTranslator.toQuery(condition, true);
+        assertNull(translation.query);
+        AggFilter aggFilter = translation.aggFilter;
+        assertEquals("InternalSqlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.in(params.a0, params.v0))",
+            aggFilter.scriptTemplate().toString());
+        assertThat(aggFilter.scriptTemplate().params().toString(), startsWith("[{a=MAX(int){a->"));
+        assertThat(aggFilter.scriptTemplate().params().toString(), endsWith(", {v=[10, null, 20, 30]}]"));
     }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
index 4a60224666515..363254f414c49 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
@@ -31,7 +31,7 @@ public void testSqlDisabled() {
         assertThat(plugin.getActions(), empty());
         assertThat(plugin.getRestHandlers(Settings.EMPTY, mock(RestController.class),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SettingsFilter(Settings.EMPTY, Collections.emptyList()),
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SettingsFilter(Collections.emptyList()),
                 mock(IndexNameExpressionResolver.class), () -> mock(DiscoveryNodes.class)), empty());
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java
index caacee0f4bada..9510b8d2213ff 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.tree;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.test.ESTestCase;
@@ -24,7 +25,7 @@
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
 import org.elasticsearch.xpack.sql.expression.gen.processor.ConstantProcessor;
 import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
-import org.elasticsearch.xpack.sql.expression.predicate.In;
+import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.sql.expression.predicate.fulltext.FullTextPredicate;
 import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InPipe;
 import org.elasticsearch.xpack.sql.expression.predicate.regex.LikePattern;
@@ -585,7 +586,7 @@ public static <T extends Node<?>> T makeNode(Class<? extends T> nodeClass) throw
     /**
      * Find all subclasses of a particular class.
      */
-    private static <T> List<Class<? extends T>> subclassesOf(Class<T> clazz) throws IOException {
+    public static <T> List<Class<? extends T>> subclassesOf(Class<T> clazz) throws IOException {
         @SuppressWarnings("unchecked") // The map is built this way
         List<Class<? extends T>> lookup = (List<Class<? extends T>>) subclassCache.get(clazz);
         if (lookup != null) {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java
index b191646a9cdee..7a04139430e33 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java
@@ -224,6 +224,12 @@ public void testConversionToNull() {
         assertNull(conversion.convert(10.0));
     }
 
+    public void testConversionFromNull() {
+        Conversion conversion = DataTypeConversion.conversionFor(DataType.NULL, DataType.INTEGER);
+        assertNull(conversion.convert(null));
+        assertNull(conversion.convert(10));
+    }
+
     public void testConversionToIdentity() {
         Conversion conversion = DataTypeConversion.conversionFor(DataType.INTEGER, DataType.INTEGER);
         assertNull(conversion.convert(null));
diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
index da67e4bdc368f..9637e64a34673 100644
--- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
+++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.http.HttpStatus;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.Request;
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json
new file mode 100644
index 0000000000000..4deaaaffc15b4
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json
@@ -0,0 +1,19 @@
+{
+  "ilm.delete_lifecycle": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "DELETE" ],
+    "url": {
+      "path": "/_ilm/policy/{policy}",
+      "paths": ["/_ilm/policy/{policy}"],
+      "parts": {
+        "policy": {
+          "type" : "string",
+          "description" : "The name of the index lifecycle policy"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json
new file mode 100644
index 0000000000000..de1b0deb5b9e0
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json
@@ -0,0 +1,24 @@
+{
+  "ilm.explain_lifecycle": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/{index}/_ilm/explain",
+      "paths": ["/{index}/_ilm/explain"],
+      "parts": {
+        "index": {
+          "type" : "string",
+          "description" : "The name of the index to explain"
+        }
+      },
+      "params": {
+        "human": {
+          "type" : "boolean",
+          "default" : "false",
+          "description" : "Return data such as dates in a human readable format"
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json
new file mode 100644
index 0000000000000..9fbabb964792e
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json
@@ -0,0 +1,19 @@
+{
+  "ilm.get_lifecycle": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_ilm/policy/{policy}",
+      "paths": ["/_ilm/policy/{policy}", "/_ilm/policy"],
+      "parts": {
+        "policy": {
+          "type" : "string",
+          "description" : "The name of the index lifecycle policy"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json
new file mode 100644
index 0000000000000..1bf2c1adf0263
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json
@@ -0,0 +1,13 @@
+{
+  "ilm.get_status": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_ilm/status",
+      "paths": ["/_ilm/status"],
+      "parts": {},
+      "params": {}
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json
new file mode 100644
index 0000000000000..ca3f1e76fb256
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json
@@ -0,0 +1,21 @@
+{
+  "ilm.move_to_step": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ilm/move/{index}",
+      "paths": ["/_ilm/move/{index}"],
+      "parts": {
+        "index": {
+          "type" : "string",
+          "description" : "The name of the index whose lifecycle step is to change"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": {
+      "description": "The new lifecycle step to move to"
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json
new file mode 100644
index 0000000000000..f23b3b3eb1537
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json
@@ -0,0 +1,21 @@
+{
+  "ilm.put_lifecycle": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "PUT" ],
+    "url": {
+      "path": "/_ilm/policy/{policy}",
+      "paths": ["/_ilm/policy/{policy}"],
+      "parts": {
+        "policy": {
+          "type" : "string",
+          "description" : "The name of the index lifecycle policy"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": {
+      "description": "The lifecycle policy definition to register"
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json
new file mode 100644
index 0000000000000..e05cd5df02282
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json
@@ -0,0 +1,19 @@
+{
+  "ilm.remove_policy": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/{index}/_ilm/remove",
+      "paths": ["/{index}/_ilm/remove", "/_ilm/remove"],
+      "parts": {
+        "index": {
+          "type" : "string",
+          "description" : "The name of the index to remove policy on"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json
new file mode 100644
index 0000000000000..af3e7fd43eb56
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json
@@ -0,0 +1,19 @@
+{
+  "ilm.retry": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/{index}/_ilm/retry",
+      "paths": ["/{index}/_ilm/retry"],
+      "parts": {
+        "index": {
+          "type" : "string",
+          "description" : "The name of the indices (comma-separated) whose failed lifecycle step is to be retry"
+        }
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json
new file mode 100644
index 0000000000000..0f2c6e347c452
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json
@@ -0,0 +1,13 @@
+{
+  "ilm.start": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ilm/start",
+      "paths": ["/_ilm/start"],
+      "parts": {},
+      "params": {}
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json
new file mode 100644
index 0000000000000..f4d6526765971
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json
@@ -0,0 +1,13 @@
+{
+  "ilm.stop": {
+    "documentation": "http://www.elastic.co/guide/en/index_lifecycle/current/index_lifecycle.html",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ilm/stop",
+      "paths": ["/_ilm/stop"],
+      "parts": {},
+      "params": {}
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.stats.json
index 40eda835a4bc8..86c982a888151 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.watcher.stats.json
@@ -8,14 +8,14 @@
       "parts": {
         "metric": {
           "type" : "enum",
-          "options" : ["_all", "queued_watches", "pending_watches"],
+          "options" : ["_all", "queued_watches", "current_watches", "pending_watches"],
           "description" : "Controls what additional stat metrics should be include in the response"
         }
       },
       "params": {
         "metric": {
           "type" : "enum",
-          "options" : ["_all", "queued_watches", "pending_watches"],
+          "options" : ["_all", "queued_watches", "current_watches", "pending_watches"],
           "description" : "Controls what additional stat metrics should be include in the response"
         },
         "emit_stacktraces": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
index c206a08e6ca91..0497f15f757c2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/custom_all_field.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml
index bf2f3bcec1cd8..6e11a1dddc71d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
index 667f80410e054..5ca3349d6c031 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_forecast.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml
index 5faba0c8031b2..3ef47c2b60880 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_job_force.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
index c13b2473cc785..e4f9bc579f48e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
index 6e9579a061339..28b5d5c9315e8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
@@ -1,6 +1,7 @@
 ---
 setup:
-
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
index e48e8e4c2f627..7f1cf3137a1ba 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test NDJSON file structure analysis without overrides":
   - do:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml
index 7f3250c7db614..2bc1a25ddad73 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeed_stats.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml
index 6daedaa8068cc..e1509084d9186 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_datafeeds.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
index e411251363b71..33da3db73cbf8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/get_model_snapshots.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
index 6a60bbb96da6f..fac725e8d5efe 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/index_layout.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test CRUD on two jobs in shared index":
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml
index d1e2851e17690..c398cc15fafe2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_groups.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
index 140078df27218..b5770dd6920b1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -399,6 +399,8 @@
 
 ---
 "Test cannot decrease model_memory_limit below current usage":
+  - skip:
+      features: headers
   - do:
       xpack.ml.put_job:
         job_id: jobs-crud-model-memory-limit-decrease
@@ -527,7 +529,8 @@
 
 ---
 "Test close job":
-
+  - skip:
+      features: headers
   - do:
       xpack.ml.put_job:
         job_id: jobs-crud-close-job
@@ -762,7 +765,8 @@
 
 ---
 "Test force close job":
-
+  - skip:
+      features: headers
   - do:
       xpack.ml.put_job:
         job_id: jobs-crud-force-close-job
@@ -929,7 +933,8 @@
 
 ---
 "Test cannot create job with existing result document":
-
+  - skip:
+      features: headers
   - do:
       headers:
         Content-Type: application/json
@@ -1068,7 +1073,8 @@
 
 ---
 "Test max model memory limit":
-
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
@@ -1314,7 +1320,8 @@
 
 ---
 "Test open job when persistent task allocation disabled":
-
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml
index 7c4903bae95cf..507b6ddd45828 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
index 125f8cbf7f8d2..6feecf82333b2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_buckets.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
index 307a1d0a80d7e..c5545120d8c60 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_categories.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
index 9b875fb1afd86..b25884da5fd00 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
index 249ff7c72d7ad..654e98c468fbe 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
index 513e1fb875774..a050ddf286297 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
index b841c8c23069f..75003bbff053d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
index 0f01613203704..576b0295abd53 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Test new fields are mapped as keyword":
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
index 68590019234a7..61f00612ffda3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
index ce638fdceaa19..5c1321a04d047 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml
index d216ecfe13e1a..48607dac7d5b6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_stop_datafeed.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: airline-data
@@ -229,7 +231,7 @@ setup:
         job_id: "start-stop-datafeed-job-field-without-mappings"
 
   - do:
-      catch: /datafeed \[start-stop-datafeed-job-field-without-mappings-feed] cannot retrieve field \[airline2\] because it has no mappings/
+      catch: /\[start-stop-datafeed-job-field-without-mappings-feed] cannot retrieve field \[airline2\] because it has no mappings/
       xpack.ml.start_datafeed:
         datafeed_id: "start-stop-datafeed-job-field-without-mappings-feed"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
index 9966ae668c08f..777738cd503e4 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
index 861be094fa62d..db485279b2bf2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
index 759ddbad2b463..4db805ae2f9f8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
index 6c9eb06d2ee9f..ad9c24357cb96 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_caps.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
index 236c0dff3ec92..f8241085f6f44 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
index cbb6f8956b14f..e0371cf5f0949 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
index 3a756efc0d72a..851d1cfa172e5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
@@ -708,5 +710,180 @@ setup:
   - match: { aggregations.histo.buckets.3.doc_count: 10 }
   - match: { aggregations.histo.buckets.3.the_max.value: 3 }
 
+---
+"Wildcards matching single rollup index":
+
+  - do:
+      xpack.rollup.rollup_search:
+        index: "foo_rollup*"
+        body:
+          size: 0
+          aggs:
+            histo:
+              date_histogram:
+                field: "timestamp"
+                interval: "1h"
+                time_zone: "UTC"
+
+  - length: { aggregations.histo.buckets: 4 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T05:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+  - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-01T06:00:00.000Z" }
+  - match: { aggregations.histo.buckets.1.doc_count: 2 }
+  - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-01T07:00:00.000Z" }
+  - match: { aggregations.histo.buckets.2.doc_count: 10 }
+  - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" }
+  - match: { aggregations.histo.buckets.3.doc_count: 20 }
+
+---
+"Wildcards matching two rollup indices":
+
+  - do:
+      indices.create:
+        index: bar
+        body:
+          mappings:
+            _doc:
+              properties:
+                timestamp:
+                  type: date
+                partition:
+                  type: keyword
+                price:
+                  type: integer
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: bar
+        body:  >
+          {
+            "index_pattern": "bar",
+            "rollup_index": "bar_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "timestamp",
+                "interval": "1h"
+              },
+              "terms": {
+                "fields": ["partition"]
+              }
+            },
+            "metrics": [
+              {
+                "field": "price",
+                "metrics": ["max"]
+              }
+            ]
+          }
+
+  - do:
+      catch: /RollupSearch currently only supports searching one rollup index at a time\./
+      xpack.rollup.rollup_search:
+        index: "*_rollup"
+        body:
+          size: 0
+          aggs:
+            histo:
+              date_histogram:
+                field: "timestamp"
+                interval: "1h"
+                time_zone: "UTC"
+
+
+---
+"Rollup search via alias":
+
+  - do:
+      indices.put_alias:
+        index: foo_rollup
+        name: rollup_alias
+
+  - do:
+      xpack.rollup.rollup_search:
+        index: "rollup_alias"
+        body:
+          size: 0
+          aggs:
+            histo:
+              date_histogram:
+                field: "timestamp"
+                interval: "1h"
+                time_zone: "UTC"
 
+  - length: { aggregations.histo.buckets: 4 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T05:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+  - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-01T06:00:00.000Z" }
+  - match: { aggregations.histo.buckets.1.doc_count: 2 }
+  - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-01T07:00:00.000Z" }
+  - match: { aggregations.histo.buckets.2.doc_count: 10 }
+  - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" }
+  - match: { aggregations.histo.buckets.3.doc_count: 20 }
+
+---
+"Rollup search via alias, multiple rollup indices match":
+
+  - do:
+      indices.create:
+        index: bar
+        body:
+          mappings:
+            _doc:
+              properties:
+                timestamp:
+                  type: date
+                partition:
+                  type: keyword
+                price:
+                  type: integer
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      xpack.rollup.put_job:
+        id: bar
+        body:  >
+          {
+            "index_pattern": "bar",
+            "rollup_index": "bar_rollup",
+            "cron": "*/30 * * * * ?",
+            "page_size" :10,
+            "groups" : {
+              "date_histogram": {
+                "field": "timestamp",
+                "interval": "1h"
+              },
+              "terms": {
+                "fields": ["partition"]
+              }
+            },
+            "metrics": [
+              {
+                "field": "price",
+                "metrics": ["max"]
+              }
+            ]
+          }
+
+  - do:
+      indices.put_alias:
+        index: foo_rollup,bar_rollup
+        name: rollup_alias
+
+  - do:
+       catch: /RollupSearch currently only supports searching one rollup index at a time\./
+       xpack.rollup.rollup_search:
+         index: "rollup_alias"
+         body:
+           size: 0
+           aggs:
+             histo:
+               date_histogram:
+                 field: "timestamp"
+                 interval: "1h"
+                 time_zone: "UTC"
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
index 57bfd821ea24d..3db0fa34ae2ea 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
@@ -6,8 +6,6 @@ setup:
       cluster.health:
           wait_for_status: yellow
 
-
-
 ---
 teardown:
   - do:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
index 38a357bcd6825..0ed11ab1b04a0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
index 849aca3332dfe..b77eb8a786adc 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
@@ -1,4 +1,6 @@
 setup:
+  - skip:
+      features: headers
   - do:
       indices.create:
         index: foo
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml
index 9844dea9135a3..369fb0129bb81 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/stats/10_basic.yml
@@ -12,3 +12,55 @@
         emit_stacktraces: "true"
   - match: { "manually_stopped": false }
   - match: { "stats.0.watcher_state": "started" }
+
+---
+"Test watcher stats current watches":
+  - skip:
+      version: " - 6.5.99"
+      reason:  metrics were fixed in 6.6.0
+
+  - do:
+      xpack.watcher.stats:
+        metric: "current_watches"
+
+  - is_false: stats.0.queued_watches
+  - is_true: stats.0.current_watches
+
+---
+"Test watcher stats queued watches":
+  - skip:
+      version: " - 6.5.99"
+      reason:  metrics were fixed in 6.6.0
+
+  - do:
+      xpack.watcher.stats:
+        metric: "queued_watches"
+
+  - is_false: stats.0.current_watches
+  - is_true: stats.0.queued_watches
+
+---
+"Test watcher stats queued watches using pending_watches":
+  - skip:
+      version: " - 6.5.99"
+      reason:  metrics were fixed in 6.6.0
+      features: warnings
+
+  - do:
+      warnings:
+        - 'The pending_watches parameter is deprecated, use queued_watches instead'
+
+      xpack.watcher.stats:
+        metric: "pending_watches"
+
+  - is_false: stats.0.current_watches
+  - is_true: stats.0.queued_watches
+
+---
+"Test watcher stats all watches":
+  - do:
+      xpack.watcher.stats:
+        metric: "_all"
+
+  - is_true: stats.0.current_watches
+  - is_true: stats.0.queued_watches
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml
index ba75c7bffc80e..428099687d7b5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/10_basic.yml
@@ -14,11 +14,12 @@
     - match:  { nodes.$master.modules.15.name: x-pack-core  }
     - match:  { nodes.$master.modules.16.name: x-pack-deprecation  }
     - match:  { nodes.$master.modules.17.name: x-pack-graph  }
-    - match:  { nodes.$master.modules.18.name: x-pack-logstash  }
-    - match:  { nodes.$master.modules.19.name: x-pack-ml  }
-    - match:  { nodes.$master.modules.20.name: x-pack-monitoring  }
-    - match:  { nodes.$master.modules.21.name: x-pack-rollup }
-    - match:  { nodes.$master.modules.22.name: x-pack-security  }
-    - match:  { nodes.$master.modules.23.name: x-pack-sql  }
-    - match:  { nodes.$master.modules.24.name: x-pack-upgrade  }
-    - match:  { nodes.$master.modules.25.name: x-pack-watcher  }
+    - match:  { nodes.$master.modules.18.name: x-pack-ilm  }
+    - match:  { nodes.$master.modules.19.name: x-pack-logstash  }
+    - match:  { nodes.$master.modules.20.name: x-pack-ml  }
+    - match:  { nodes.$master.modules.21.name: x-pack-monitoring  }
+    - match:  { nodes.$master.modules.22.name: x-pack-rollup }
+    - match:  { nodes.$master.modules.23.name: x-pack-security  }
+    - match:  { nodes.$master.modules.24.name: x-pack-sql  }
+    - match:  { nodes.$master.modules.25.name: x-pack-upgrade  }
+    - match:  { nodes.$master.modules.26.name: x-pack-watcher  }
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
index d91cc2a6f0148..2b46a661e17b5 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeCheck.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.protocol.xpack.migration.UpgradeActionRequired;
 import org.elasticsearch.script.Script;
@@ -43,17 +42,16 @@ public class IndexUpgradeCheck<T> extends AbstractComponent {
      * Creates a new upgrade check
      *
      * @param name           - the name of the check
-     * @param settings       - system settings
      * @param actionRequired - return true if they can work with the index with specified name
      * @param client         - client
      * @param clusterService - cluster service
      * @param types          - a list of types that the reindexing should be limited to
      * @param updateScript   - the upgrade script that should be used during reindexing
      */
-    public IndexUpgradeCheck(String name, Settings settings,
+    public IndexUpgradeCheck(String name,
                              Function<IndexMetaData, UpgradeActionRequired> actionRequired,
                              Client client, ClusterService clusterService, String[] types, Script updateScript) {
-        this(name, settings, actionRequired, client, clusterService, types, updateScript,
+        this(name, actionRequired, client, clusterService, types, updateScript,
                 listener -> listener.onResponse(null), (t, listener) -> listener.onResponse(TransportResponse.Empty.INSTANCE));
     }
 
@@ -61,7 +59,6 @@ public IndexUpgradeCheck(String name, Settings settings,
      * Creates a new upgrade check
      *
      * @param name           - the name of the check
-     * @param settings       - system settings
      * @param actionRequired - return true if they can work with the index with specified name
      * @param client         - client
      * @param clusterService - cluster service
@@ -70,12 +67,11 @@ public IndexUpgradeCheck(String name, Settings settings,
      * @param preUpgrade     - action that should be performed before upgrade
      * @param postUpgrade    - action that should be performed after upgrade
      */
-    public IndexUpgradeCheck(String name, Settings settings,
+    public IndexUpgradeCheck(String name,
                              Function<IndexMetaData, UpgradeActionRequired> actionRequired,
                              Client client, ClusterService clusterService, String[] types, Script updateScript,
                              Consumer<ActionListener<T>> preUpgrade,
                              BiConsumer<T, ActionListener<TransportResponse.Empty>> postUpgrade) {
-        super(settings);
         this.name = name;
         this.actionRequired = actionRequired;
         this.reindexer = new InternalIndexReindexer<>(client, clusterService, IndexUpgradeCheckVersion.UPRADE_VERSION, updateScript,
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
index 07017e6fc0014..e1fec4f8f002c 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/IndexUpgradeService.java
@@ -30,7 +30,6 @@ public class IndexUpgradeService extends AbstractComponent {
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     public IndexUpgradeService(Settings settings, List<IndexUpgradeCheck> upgradeChecks) {
-        super(settings);
         this.upgradeChecks = upgradeChecks;
         this.indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
     }
diff --git a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java
index d5c6689a9c388..80d4a17eb5b8a 100644
--- a/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java
+++ b/x-pack/plugin/upgrade/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java
@@ -144,7 +144,6 @@ static BiFunction<Client, ClusterService, IndexUpgradeCheck> getSecurityUpgradeC
         return (client, clusterService) -> {
             final Client clientWithOrigin = clientWithOrigin(client, SECURITY_ORIGIN);
             return new IndexUpgradeCheck<Void>("security",
-                    settings,
                     indexMetaData -> {
                         if (".security".equals(indexMetaData.getIndex().getName())
                                 || indexMetaData.getAliases().containsKey(".security")) {
@@ -244,7 +243,6 @@ static BiFunction<Client, ClusterService, IndexUpgradeCheck> getWatchesIndexUpgr
         return (client, clusterService) -> {
             final Client clientWithOrigin = clientWithOrigin(client, WATCHER_ORIGIN);
             return new IndexUpgradeCheck<Boolean>("watches",
-                    settings,
                     indexMetaData -> {
                         if (indexOrAliasExists(indexMetaData, ".watches")) {
                             if (checkInternalIndexFormat(indexMetaData)) {
@@ -273,7 +271,6 @@ static BiFunction<Client, ClusterService, IndexUpgradeCheck> getTriggeredWatches
         return (client, clusterService) -> {
             final Client clientWithOrigin = clientWithOrigin(client, WATCHER_ORIGIN);
             return new IndexUpgradeCheck<Boolean>("triggered-watches",
-                    settings,
                     indexMetaData -> {
                         if (indexOrAliasExists(indexMetaData, TriggeredWatchStoreField.INDEX_NAME)) {
                             if (checkInternalIndexFormat(indexMetaData)) {
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
index 0e06efe79d80f..169e2c256b6f3 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeIT.java
@@ -88,7 +88,7 @@ public void testInternalUpgradePrePostChecks() throws Exception {
         AtomicBoolean postUpgradeIsCalled = new AtomicBoolean();
 
         IndexUpgradeCheck check = new IndexUpgradeCheck<Long>(
-                "test", Settings.EMPTY,
+                "test",
                 indexMetaData -> {
                     if (indexMetaData.getIndex().getName().equals(testIndex)) {
                         return UpgradeActionRequired.UPGRADE;
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
index 090d7d11baf76..1bb7b395100d0 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeServiceTests.java
@@ -25,7 +25,7 @@
 
 public class IndexUpgradeServiceTests extends ESTestCase {
 
-    private IndexUpgradeCheck upgradeBarCheck = new IndexUpgradeCheck("upgrade_bar", Settings.EMPTY,
+    private IndexUpgradeCheck upgradeBarCheck = new IndexUpgradeCheck("upgrade_bar",
             (Function<IndexMetaData, UpgradeActionRequired>) indexMetaData -> {
                 if ("bar".equals(indexMetaData.getSettings().get("test.setting"))) {
                     return UpgradeActionRequired.UPGRADE;
@@ -34,7 +34,7 @@ public class IndexUpgradeServiceTests extends ESTestCase {
                 }
             }, null, null, null, null);
 
-    private IndexUpgradeCheck reindexFooCheck = new IndexUpgradeCheck("reindex_foo", Settings.EMPTY,
+    private IndexUpgradeCheck reindexFooCheck = new IndexUpgradeCheck("reindex_foo",
             (Function<IndexMetaData, UpgradeActionRequired>) indexMetaData -> {
                 if ("foo".equals(indexMetaData.getSettings().get("test.setting"))) {
                     return UpgradeActionRequired.REINDEX;
@@ -43,10 +43,10 @@ public class IndexUpgradeServiceTests extends ESTestCase {
                 }
             }, null, null, null, null);
 
-    private IndexUpgradeCheck everythingIsFineCheck = new IndexUpgradeCheck("everything_is_fine", Settings.EMPTY,
+    private IndexUpgradeCheck everythingIsFineCheck = new IndexUpgradeCheck("everything_is_fine",
             indexMetaData -> UpgradeActionRequired.UP_TO_DATE, null, null, null, null);
 
-    private IndexUpgradeCheck unreachableCheck = new IndexUpgradeCheck("unreachable", Settings.EMPTY,
+    private IndexUpgradeCheck unreachableCheck = new IndexUpgradeCheck("unreachable",
             (Function<IndexMetaData, UpgradeActionRequired>) indexMetaData -> {
                 fail("Unreachable check is called");
                 return null;
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
index 6b624de6c6f76..e30764c0c109c 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeTasksIT.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.upgrade;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -16,7 +17,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -98,7 +98,7 @@ public String pluginScriptLang() {
         public MockUpgradePlugin(Settings settings) {
             this.settings = settings;
             this.upgrade = new Upgrade(settings);
-            Loggers.getLogger(IndexUpgradeTasksIT.class).info("MockUpgradePlugin is created");
+            LogManager.getLogger(IndexUpgradeTasksIT.class).info("MockUpgradePlugin is created");
         }
 
 
@@ -122,7 +122,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                                                    NamedXContentRegistry xContentRegistry, Environment environment,
                                                    NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
             return Arrays.asList(new IndexUpgradeService(settings, Collections.singletonList(
-                    new IndexUpgradeCheck("test", settings,
+                    new IndexUpgradeCheck("test",
                             new Function<IndexMetaData, UpgradeActionRequired>() {
                                 @Override
                                 public UpgradeActionRequired apply(IndexMetaData indexMetaData) {
diff --git a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeWatcherIT.java b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeWatcherIT.java
index 1d9d655642aee..1c0fcc71fa0ea 100644
--- a/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeWatcherIT.java
+++ b/x-pack/plugin/upgrade/src/test/java/org/elasticsearch/xpack/upgrade/IndexUpgradeWatcherIT.java
@@ -304,10 +304,9 @@ private ActionListener<Boolean> createLatchListener(CountDownLatch latch,
     private void triggerClusterStateEvent() {
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
         Client client = clientWithOrigin(internalCluster().getInstance(Client.class, internalCluster().getMasterName()), WATCHER_ORIGIN);
-        Settings settings = internalCluster().getInstance(Settings.class, internalCluster().getMasterName());
         ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
         WatcherIndexTemplateRegistry registry =
-                new WatcherIndexTemplateRegistry(settings, clusterService, threadPool, client);
+                new WatcherIndexTemplateRegistry(clusterService, threadPool, client);
 
         ClusterState state = clusterService.state();
         ClusterChangedEvent event = new ClusterChangedEvent("whatever", state, state);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index eb2652ed27450..ca04ce528aae5 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -295,7 +295,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
             throw new UncheckedIOException(e);
         }
 
-        new WatcherIndexTemplateRegistry(settings, clusterService, threadPool, client);
+        new WatcherIndexTemplateRegistry(clusterService, threadPool, client);
 
         // http client
         httpClient = new HttpClient(settings, getSslService(), cryptoService);
@@ -313,7 +313,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         reloadableServices.add(slackService);
         reloadableServices.add(pagerDutyService);
 
-        TextTemplateEngine templateEngine = new TextTemplateEngine(settings, scriptService);
+        TextTemplateEngine templateEngine = new TextTemplateEngine(scriptService);
         Map<String, EmailAttachmentParser> emailAttachmentParsers = new HashMap<>();
         emailAttachmentParsers.put(HttpEmailAttachementParser.TYPE, new HttpEmailAttachementParser(httpClient, templateEngine));
         emailAttachmentParsers.put(DataAttachmentParser.TYPE, new DataAttachmentParser());
@@ -399,7 +399,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
             .setConcurrentRequests(SETTING_BULK_CONCURRENT_REQUESTS.get(settings))
             .build();
 
-        HistoryStore historyStore = new HistoryStore(settings, bulkProcessor);
+        HistoryStore historyStore = new HistoryStore(bulkProcessor);
 
         // schedulers
         final Set<Schedule.Parser> scheduleParsers = new HashSet<>();
@@ -418,15 +418,15 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
         final Set<TriggerEngine> triggerEngines = new HashSet<>();
         triggerEngines.add(manualTriggerEngine);
         triggerEngines.add(configuredTriggerEngine);
-        final TriggerService triggerService = new TriggerService(settings, triggerEngines);
+        final TriggerService triggerService = new TriggerService(triggerEngines);
 
-        final TriggeredWatch.Parser triggeredWatchParser = new TriggeredWatch.Parser(settings, triggerService);
+        final TriggeredWatch.Parser triggeredWatchParser = new TriggeredWatch.Parser(triggerService);
         final TriggeredWatchStore triggeredWatchStore = new TriggeredWatchStore(settings, client, triggeredWatchParser, bulkProcessor);
 
         final WatcherSearchTemplateService watcherSearchTemplateService =
-                new WatcherSearchTemplateService(settings, scriptService, xContentRegistry);
+                new WatcherSearchTemplateService(scriptService, xContentRegistry);
         final WatchExecutor watchExecutor = getWatchExecutor(threadPool);
-        final WatchParser watchParser = new WatchParser(settings, triggerService, registry, inputRegistry, cryptoService, getClock());
+        final WatchParser watchParser = new WatchParser(triggerService, registry, inputRegistry, cryptoService, getClock());
 
         final ExecutionService executionService = new ExecutionService(settings, historyStore, triggeredWatchStore, watchExecutor,
                 getClock(), watchParser, clusterService, client, threadPool.generic());
@@ -440,7 +440,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
         final WatcherLifeCycleService watcherLifeCycleService =
                 new WatcherLifeCycleService(settings, clusterService, watcherService);
 
-        listener = new WatcherIndexingListener(settings, watchParser, getClock(), triggerService);
+        listener = new WatcherIndexingListener(watchParser, getClock(), triggerService);
         clusterService.addListener(listener);
 
         return Arrays.asList(registry, inputRegistry, historyStore, triggerService, triggeredWatchParser,
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
index 622175b46daba..e744712f6f9d3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
@@ -65,8 +64,7 @@ final class WatcherIndexingListener extends AbstractComponent implements Indexin
     private final TriggerService triggerService;
     private volatile Configuration configuration = INACTIVE;
 
-    WatcherIndexingListener(Settings settings, WatchParser parser, Clock clock, TriggerService triggerService) {
-        super(settings);
+    WatcherIndexingListener(WatchParser parser, Clock clock, TriggerService triggerService) {
         this.parser = parser;
         this.clock = clock;
         this.triggerService = triggerService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
index e2246214315b8..1e62257576b7d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
@@ -54,7 +54,6 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
     private volatile WatcherService watcherService;
 
     WatcherLifeCycleService(Settings settings, ClusterService clusterService, WatcherService watcherService) {
-        super(settings);
         this.watcherService = watcherService;
         this.requireManualStart = SETTING_REQUIRE_MANUAL_START.get(settings);
         clusterService.addListener(this);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
index 75fd13915de3e..f26ab5a14fb06 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
@@ -83,7 +83,6 @@ public class WatcherService extends AbstractComponent {
 
     WatcherService(Settings settings, TriggerService triggerService, TriggeredWatchStore triggeredWatchStore,
                    ExecutionService executionService, WatchParser parser, Client client, ExecutorService executor) {
-        super(settings);
         this.triggerService = triggerService;
         this.triggeredWatchStore = triggeredWatchStore;
         this.executionService = executionService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
index cd682260ad9a4..2a327851558e5 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
@@ -75,11 +75,10 @@ public class HttpClient extends AbstractComponent implements Closeable {
     private final CryptoService cryptoService;
 
     public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) {
-        super(settings);
         this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings);
         this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings);
         this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings);
-        this.settingsProxy = getProxyFromSettings();
+        this.settingsProxy = getProxyFromSettings(settings);
         this.cryptoService = cryptoService;
 
         HttpClientBuilder clientBuilder = HttpClientBuilder.create();
@@ -228,7 +227,7 @@ static void setProxy(RequestConfig.Builder config, HttpRequest request, HttpProx
      *
      * @return An HTTP proxy instance, if no settings are configured this will be an HttpProxy.NO_PROXY instance
      */
-    private HttpProxy getProxyFromSettings() {
+    private HttpProxy getProxyFromSettings(Settings settings) {
         String proxyHost = HttpSettings.PROXY_HOST.get(settings);
         Scheme proxyScheme = HttpSettings.PROXY_SCHEME.exists(settings) ?
                 Scheme.parse(HttpSettings.PROXY_SCHEME.get(settings)) : Scheme.HTTP;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
index d2159fd572f5d..1e1188ea55d41 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateEngine.java
@@ -5,8 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.common.text;
 
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
@@ -17,12 +15,11 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class TextTemplateEngine extends AbstractComponent {
+public class TextTemplateEngine {
 
     private final ScriptService service;
 
-    public TextTemplateEngine(Settings settings, ScriptService service) {
-        super(settings);
+    public TextTemplateEngine(ScriptService service) {
         this.service = service;
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/AbstractCompareCondition.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/AbstractCompareCondition.java
index 81e3eb464e69b..7b0c176927d8d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/AbstractCompareCondition.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/AbstractCompareCondition.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.xpack.core.watcher.condition.ExecutableCondition;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.support.Variables;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/ArrayCompareCondition.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/ArrayCompareCondition.java
index 6aeccf734c2f4..5f4ee988012ae 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/ArrayCompareCondition.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/ArrayCompareCondition.java
@@ -8,8 +8,8 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.watcher.common.xcontent.XContentUtils;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.XContentUtils;
+import org.elasticsearch.common.xcontent.ObjectPath;
 
 import java.io.IOException;
 import java.time.Clock;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/CompareCondition.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/CompareCondition.java
index a3b21dec295d2..7c990e6582ea2 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/CompareCondition.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/condition/CompareCondition.java
@@ -8,8 +8,8 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.watcher.common.xcontent.XContentUtils;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.XContentUtils;
+import org.elasticsearch.common.xcontent.ObjectPath;
 
 import java.io.IOException;
 import java.time.Clock;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java
index bbaeba8ad0449..03289df73fc99 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java
@@ -100,7 +100,6 @@ public class ExecutionService extends AbstractComponent {
     public ExecutionService(Settings settings, HistoryStore historyStore, TriggeredWatchStore triggeredWatchStore, WatchExecutor executor,
                             Clock clock, WatchParser parser, ClusterService clusterService, Client client,
                             ExecutorService genericExecutor) {
-        super(settings);
         this.historyStore = historyStore;
         this.triggeredWatchStore = triggeredWatchStore;
         this.executor = executor;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatch.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatch.java
index 51998a14bd770..91475f499714e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatch.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatch.java
@@ -8,8 +8,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -48,12 +46,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public static class Parser extends AbstractComponent {
+    public static class Parser {
 
         private final TriggerService triggerService;
 
-        public Parser(Settings settings, TriggerService triggerService) {
-            super(settings);
+        public Parser(TriggerService triggerService) {
             this.triggerService = triggerService;
         }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
index 9a4b555d63355..049a7ff200e77 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStore.java
@@ -60,7 +60,6 @@ public class TriggeredWatchStore extends AbstractComponent {
     private final BulkProcessor bulkProcessor;
 
     public TriggeredWatchStore(Settings settings, Client client, TriggeredWatch.Parser triggeredWatchParser, BulkProcessor bulkProcessor) {
-        super(settings);
         this.scrollSize = settings.getAsInt("xpack.watcher.execution.scroll.size", 1000);
         this.client = ClientHelper.clientWithOrigin(client, WATCHER_ORIGIN);
         this.scrollTimeout = settings.getAsTime("xpack.watcher.execution.scroll.timeout", TimeValue.timeValueMinutes(5));
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
index 723568f8ba75d..a20eef6b4895b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
@@ -32,8 +31,7 @@ public class HistoryStore extends AbstractComponent {
 
     private final BulkProcessor bulkProcessor;
 
-    public HistoryStore(Settings settings, BulkProcessor bulkProcessor) {
-        super(settings);
+    public HistoryStore(BulkProcessor bulkProcessor) {
         this.bulkProcessor = bulkProcessor;
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java
index e6c78f6ac0203..9ca790c3df5b8 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java
@@ -26,7 +26,7 @@ public SearchInputFactory(Settings settings, Client client, NamedXContentRegistr
                               ScriptService scriptService) {
         this.client = client;
         this.defaultTimeout = settings.getAsTime("xpack.watcher.input.search.default_timeout", TimeValue.timeValueMinutes(1));
-        this.searchTemplateService = new WatcherSearchTemplateService(settings, scriptService, xContentRegistry);
+        this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
index 027825ab77871..0b545c7942821 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
@@ -28,15 +28,14 @@ public abstract class NotificationService<Account> extends AbstractComponent {
     private Map<String, Account> accounts;
     private Account defaultAccount;
 
-    public NotificationService(Settings settings, String type,
+    public NotificationService(String type,
                                ClusterSettings clusterSettings, List<Setting<?>> pluginSettings) {
-        this(settings, type);
+        this(type);
         clusterSettings.addSettingsUpdateConsumer(this::reload, pluginSettings);
     }
 
     // Used for testing only
-    NotificationService(Settings settings, String type) {
-        super(settings);
+    NotificationService(String type) {
         this.type = type;
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
index e45ed55cee3ac..70922e57bd078 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java
@@ -104,7 +104,7 @@ public class EmailService extends NotificationService<Account> {
     private final CryptoService cryptoService;
 
     public EmailService(Settings settings, @Nullable CryptoService cryptoService, ClusterSettings clusterSettings) {
-        super(settings, "email", clusterSettings, EmailService.getSettings());
+        super("email", clusterSettings, EmailService.getSettings());
         this.cryptoService = cryptoService;
         // ensure logging of setting changes
         clusterSettings.addSettingsUpdateConsumer(SETTING_DEFAULT_ACCOUNT, (s) -> {});
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
index f6026c0efce3b..d0d9ecc78104c 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParser.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.watcher.notification.email.attachment;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -22,13 +22,13 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpProxy;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
-import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.notification.email.Attachment;
@@ -74,7 +74,7 @@ public ReportingAttachmentParser(Settings settings, HttpClient httpClient, TextT
         this.retries = RETRIES_SETTING.get(settings);
         this.httpClient = httpClient;
         this.templateEngine = templateEngine;
-        this.logger = Loggers.getLogger(getClass());
+        this.logger = LogManager.getLogger(getClass());
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
index 926a8940ad0e6..57e9394a0e3e7 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/hipchat/HipChatService.java
@@ -64,7 +64,7 @@ public class HipChatService extends NotificationService<HipChatAccount> {
     private HipChatServer defaultServer;
 
     public HipChatService(Settings settings, HttpClient httpClient, ClusterSettings clusterSettings) {
-        super(settings, "hipchat", clusterSettings, HipChatService.getSettings());
+        super("hipchat", clusterSettings, HipChatService.getSettings());
         this.httpClient = httpClient;
         // ensure logging of setting changes
         clusterSettings.addSettingsUpdateConsumer(SETTING_DEFAULT_ACCOUNT, (s) -> {});
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
index 4034f2d2202ee..b89aafd2ab2c0 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraService.java
@@ -62,7 +62,7 @@ public class JiraService extends NotificationService<JiraAccount> {
     private final HttpClient httpClient;
 
     public JiraService(Settings settings, HttpClient httpClient, ClusterSettings clusterSettings) {
-        super(settings, "jira", clusterSettings, JiraService.getSettings());
+        super("jira", clusterSettings, JiraService.getSettings());
         this.httpClient = httpClient;
         // ensure logging of setting changes
         clusterSettings.addSettingsUpdateConsumer(SETTING_DEFAULT_ACCOUNT, (s) -> {});
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
index d1622757f6a01..d646d62e8750e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/PagerDutyService.java
@@ -40,7 +40,7 @@ public class PagerDutyService extends NotificationService<PagerDutyAccount> {
     private final HttpClient httpClient;
 
     public PagerDutyService(Settings settings, HttpClient httpClient, ClusterSettings clusterSettings) {
-        super(settings, "pagerduty", clusterSettings, PagerDutyService.getSettings());
+        super("pagerduty", clusterSettings, PagerDutyService.getSettings());
         this.httpClient = httpClient;
         clusterSettings.addSettingsUpdateConsumer(SETTING_DEFAULT_ACCOUNT, (s) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_SERVICE_API_KEY, (s, o) -> {}, (s, o) -> {});
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
index 49fb6a831f62c..23190adb37d38 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SlackService.java
@@ -40,7 +40,7 @@ public class SlackService extends NotificationService<SlackAccount> {
     private final HttpClient httpClient;
 
     public SlackService(Settings settings, HttpClient httpClient, ClusterSettings clusterSettings) {
-        super(settings, "slack", clusterSettings, SlackService.getSettings());
+        super("slack", clusterSettings, SlackService.getSettings());
         this.httpClient = httpClient;
         clusterSettings.addSettingsUpdateConsumer(SETTING_DEFAULT_ACCOUNT, (s) -> {});
         clusterSettings.addAffixUpdateConsumer(SETTING_URL, (s, o) -> {}, (s, o) -> {});
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/DynamicAttachments.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/DynamicAttachments.java
index 4c53fc767c788..3a5eae2eb56dd 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/DynamicAttachments.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/DynamicAttachments.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
index 9f6aaa05ef7a7..dd84dcb1d086b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
@@ -51,8 +51,12 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest restRequest, Wa
             request.includeCurrentWatches(true);
             request.includeQueuedWatches(true);
         } else {
-            request.includeCurrentWatches(metrics.contains("queued_watches"));
-            request.includeQueuedWatches(metrics.contains("pending_watches"));
+            request.includeCurrentWatches(metrics.contains("current_watches"));
+            request.includeQueuedWatches(metrics.contains("queued_watches") || metrics.contains("pending_watches"));
+        }
+
+        if (metrics.contains("pending_watches")) {
+            deprecationLogger.deprecated("The pending_watches parameter is deprecated, use queued_watches instead");
         }
 
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
index 36ec856beddff..6d0a02062ee17 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.GatewayService;
@@ -51,8 +50,7 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
     private final TemplateConfig[] indexTemplates;
     private final ConcurrentMap<String, AtomicBoolean> templateCreationsInProgress = new ConcurrentHashMap<>();
 
-    public WatcherIndexTemplateRegistry(Settings settings, ClusterService clusterService, ThreadPool threadPool, Client client) {
-        super(settings);
+    public WatcherIndexTemplateRegistry(ClusterService clusterService, ThreadPool threadPool, Client client) {
         this.client = client;
         this.threadPool = threadPool;
         this.indexTemplates = TEMPLATE_CONFIGS;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
index 2208aab428a39..d8451fbc7b3b3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
@@ -8,7 +8,6 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -36,8 +35,7 @@ public class WatcherSearchTemplateService extends AbstractComponent {
     private final ScriptService scriptService;
     private final NamedXContentRegistry xContentRegistry;
 
-    public WatcherSearchTemplateService(Settings settings, ScriptService scriptService, NamedXContentRegistry xContentRegistry) {
-        super(settings);
+    public WatcherSearchTemplateService(ScriptService scriptService, NamedXContentRegistry xContentRegistry) {
         this.scriptService = scriptService;
         this.xContentRegistry = xContentRegistry;
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java
index 862936688727c..72b91d0756240 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transform/search/SearchTransformFactory.java
@@ -27,7 +27,7 @@ public SearchTransformFactory(Settings settings, Client client, NamedXContentReg
         super(LogManager.getLogger(ExecutableSearchTransform.class));
         this.client = client;
         this.defaultTimeout = settings.getAsTime("xpack.watcher.transform.search.default_timeout", TimeValue.timeValueMinutes(1));
-        this.searchTemplateService = new WatcherSearchTemplateService(settings, scriptService, xContentRegistry);
+        this.searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerService.java
index 355d6d6c32e41..c9d99a2e2c4ff 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/TriggerService.java
@@ -8,7 +8,6 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
 import org.elasticsearch.xpack.core.watcher.trigger.Trigger;
@@ -33,8 +32,7 @@ public class TriggerService extends AbstractComponent {
     private final Map<String, TriggerEngine> engines;
     private final Map<String, TriggerWatchStats> perWatchStats = new HashMap<>();
 
-    public TriggerService(Settings settings, Set<TriggerEngine> engines) {
-        super(settings);
+    public TriggerService(Set<TriggerEngine> engines) {
         Map<String, TriggerEngine> builder = new HashMap<>();
         for (TriggerEngine engine : engines) {
             builder.put(engine.type(), engine);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTriggerEngine.java
index dfee54c391aff..c53162bd7d9c8 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTriggerEngine.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleTriggerEngine.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.support.WatcherDateTimeUtils;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
@@ -33,8 +32,7 @@ public abstract class ScheduleTriggerEngine extends AbstractComponent implements
     protected final ScheduleRegistry scheduleRegistry;
     protected final Clock clock;
 
-    public ScheduleTriggerEngine(Settings settings, ScheduleRegistry scheduleRegistry, Clock clock) {
-        super(settings);
+    public ScheduleTriggerEngine(ScheduleRegistry scheduleRegistry, Clock clock) {
         this.scheduleRegistry = scheduleRegistry;
         this.clock = clock;
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
index bd0204766aff4..2fe74be2b0c2e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/engine/TickerScheduleTriggerEngine.java
@@ -41,7 +41,7 @@ public class TickerScheduleTriggerEngine extends ScheduleTriggerEngine {
     private Ticker ticker;
 
     public TickerScheduleTriggerEngine(Settings settings, ScheduleRegistry scheduleRegistry, Clock clock) {
-        super(settings, scheduleRegistry, clock);
+        super(scheduleRegistry, clock);
         this.tickInterval = TICKER_INTERVAL_SETTING.get(settings);
         this.schedules = new ConcurrentHashMap<>();
         this.ticker = new Ticker(Node.NODE_DATA_SETTING.get(settings));
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java
index 9f9dfc8313477..bc65335e844a0 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/watch/WatchParser.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -61,10 +60,8 @@ public class WatchParser extends AbstractComponent {
     private final List<ActionWrapper> defaultActions;
     private final Clock clock;
 
-    public WatchParser(Settings settings, TriggerService triggerService, ActionRegistry actionRegistry, InputRegistry inputRegistry,
+    public WatchParser(TriggerService triggerService, ActionRegistry actionRegistry, InputRegistry inputRegistry,
                        @Nullable CryptoService cryptoService, Clock clock) {
-
-        super(settings);
         this.triggerService = triggerService;
         this.actionRegistry = actionRegistry;
         this.inputRegistry = inputRegistry;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java
index e1e8b5b2ddd7d..46b0fc0c30e7b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
index 4ff36d3af9392..aca6fb8d03b1b 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
@@ -87,7 +87,7 @@ public class WatcherIndexingListenerTests extends ESTestCase {
     @Before
     public void setup() throws Exception {
         clock.freeze();
-        listener = new WatcherIndexingListener(Settings.EMPTY, parser, clock, triggerService);
+        listener = new WatcherIndexingListener(parser, clock, triggerService);
 
         Map<ShardId, ShardAllocationConfiguration> map = new HashMap<>();
         map.put(shardId, new ShardAllocationConfiguration(0, 1, Collections.singletonList("foo")));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
index 73ad045652435..0f670ea4cde9e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
@@ -223,7 +223,7 @@ public void testPausingWatcherServiceAlsoPausesTriggerService() {
         String engineType = "foo";
         TriggerEngine triggerEngine = mock(TriggerEngine.class);
         when(triggerEngine.type()).thenReturn(engineType);
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, Collections.singleton(triggerEngine));
+        TriggerService triggerService = new TriggerService(Collections.singleton(triggerEngine));
 
         Trigger trigger = mock(Trigger.class);
         when(trigger.type()).thenReturn(engineType);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
index 9b851131d7d6d..a753bf05b8229 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import java.util.Map;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
index aa03bf4545e5b..c4604d8e2a14d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/ExecutableJiraActionTests.java
@@ -300,7 +300,7 @@ private WatchExecutionContext createWatchExecutionContext() {
     class UpperCaseTextTemplateEngine extends TextTemplateEngine {
 
         UpperCaseTextTemplateEngine() {
-            super(Settings.EMPTY, mock(ScriptService.class));
+            super(mock(ScriptService.class));
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionTests.java
index 22a6ced9e7e08..a0d09e39c029e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionTests.java
@@ -293,7 +293,7 @@ class ModelTextTemplateEngine extends TextTemplateEngine {
         private final Map<String, Object> model;
 
         ModelTextTemplateEngine(Map<String, Object> model) {
-            super(Settings.EMPTY, mock(ScriptService.class));
+            super(mock(ScriptService.class));
             this.model = model;
         }
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
index 5856f0fa92082..8b5994b8cf395 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
index 002d833c20913..ecc071d598105 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/text/TextTemplateTests.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.watcher.common.text;
 
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -47,7 +46,7 @@ public class TextTemplateTests extends ESTestCase {
     @Before
     public void init() throws Exception {
         service = mock(ScriptService.class);
-        engine = new TextTemplateEngine(Settings.EMPTY, service);
+        engine = new TextTemplateEngine(service);
     }
 
     public void testRender() throws Exception {
@@ -131,7 +130,7 @@ private void assertNoCompilation(String input) {
 
     private void assertScriptServiceInvoked(final String input) {
         ScriptService scriptService = mock(ScriptService.class);
-        TextTemplateEngine e = new TextTemplateEngine(Settings.EMPTY, scriptService);
+        TextTemplateEngine e = new TextTemplateEngine(scriptService);
 
         TemplateScript.Factory compiledTemplate = templateParams ->
             new TemplateScript(templateParams) {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
index ff02fe1b06fc0..6b292be875575 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
@@ -53,7 +53,7 @@
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
 import org.elasticsearch.xpack.core.watcher.input.ExecutableInput;
 import org.elasticsearch.xpack.core.watcher.input.Input;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transform.ExecutableTransform;
 import org.elasticsearch.xpack.core.watcher.transform.Transform;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionTests.java
index 6b6c8d9b1375e..a31de21073ca8 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ManualExecutionTests.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.plugins.Plugin;
@@ -20,7 +21,6 @@
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
index 4012c8d24b5b5..428ec96df97e0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
@@ -398,10 +398,10 @@ public void testTriggeredWatchParser() throws Exception {
         triggeredWatch.toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS);
 
         ScheduleRegistry scheduleRegistry = new ScheduleRegistry(Collections.singleton(new CronSchedule.Parser()));
-        TriggerEngine triggerEngine = new WatchTests.ParseOnlyScheduleTriggerEngine(Settings.EMPTY, scheduleRegistry, new ClockMock());
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, singleton(triggerEngine));
+        TriggerEngine triggerEngine = new WatchTests.ParseOnlyScheduleTriggerEngine(scheduleRegistry, new ClockMock());
+        TriggerService triggerService = new TriggerService(singleton(triggerEngine));
 
-        TriggeredWatch.Parser parser = new TriggeredWatch.Parser(Settings.EMPTY, triggerService);
+        TriggeredWatch.Parser parser = new TriggeredWatch.Parser(triggerService);
         TriggeredWatch parsedTriggeredWatch = parser.parse(triggeredWatch.id().value(), 0, BytesReference.bytes(jsonBuilder));
 
         XContentBuilder jsonBuilder2 = XContentFactory.jsonBuilder();
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
index 19bf1ba5a1fd3..2ea364de18b4e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
@@ -71,7 +71,7 @@ public void init() {
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(settings));
         BulkProcessor.Listener listener = mock(BulkProcessor.Listener.class);
         BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener).setConcurrentRequests(0).setBulkActions(1).build();
-        historyStore = new HistoryStore(settings, bulkProcessor);
+        historyStore = new HistoryStore(bulkProcessor);
     }
 
     public void testPut() throws Exception {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
index a5de71b240091..20720d6703db9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
index fb957d736e1ee..389f14a06f930 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.watcher.common.http.HttpClient;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
index cb86913678a96..ddf45de816367 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
@@ -81,7 +81,7 @@ public void testAccountDoesNotExist() throws Exception{
     private static class TestNotificationService extends NotificationService<String> {
 
         TestNotificationService(Settings settings) {
-            super(settings, "test");
+            super("test");
             reload(settings);
         }
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
index 97907bbef8a59..d46524b93e508 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
@@ -352,7 +352,7 @@ public void testThatUrlIsTemplatable() throws Exception {
                 .thenReturn(new HttpResponse(503))
                 .thenReturn(new HttpResponse(200, randomAlphaOfLength(10)));
 
-        TextTemplateEngine replaceHttpWithHttpsTemplateEngine = new TextTemplateEngine(Settings.EMPTY, null) {
+        TextTemplateEngine replaceHttpWithHttpsTemplateEngine = new TextTemplateEngine(null) {
             @Override
             public String render(TextTemplate textTemplate, Map<String, Object> model) {
                 return textTemplate.getTemplate().replaceAll("REPLACEME", "REPLACED");
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/FilterXContentTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/FilterXContentTests.java
index eadf739b175a3..fbe0b98f4ca63 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/FilterXContentTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/FilterXContentTests.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
index 3201a69b486d3..e93a86b93eb23 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
@@ -72,7 +72,7 @@ public void createRegistryAndClient() {
         }).when(indicesAdminClient).putTemplate(any(PutIndexTemplateRequest.class), any(ActionListener.class));
 
         ClusterService clusterService = mock(ClusterService.class);
-        registry = new WatcherIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client);
+        registry = new WatcherIndexTemplateRegistry(clusterService, threadPool, client);
     }
 
     public void testThatNonExistingTemplatesAreAddedImmediately() {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/MockTextTemplateEngine.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/MockTextTemplateEngine.java
index eeefa20c3a158..a8db182bda054 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/MockTextTemplateEngine.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/MockTextTemplateEngine.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.test;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 
@@ -13,7 +12,7 @@
 
 public class MockTextTemplateEngine extends TextTemplateEngine {
     public MockTextTemplateEngine() {
-        super(Settings.EMPTY, null);
+        super(null);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
index 99cf45e583dc7..23f7e6e7c79b5 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
@@ -59,7 +59,7 @@ protected Clock getClock() {
 
             @Override
             protected TriggerEngine getTriggerEngine(Clock clock, ScheduleRegistry scheduleRegistry){
-                return new ScheduleTriggerEngineMock(settings, scheduleRegistry, clock);
+                return new ScheduleTriggerEngineMock(scheduleRegistry, clock);
             }
 
             @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
index 25ae9bb30ac14..318d9078bc667 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -28,7 +29,6 @@
 import org.elasticsearch.xpack.core.watcher.common.secret.Secret;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.core.watcher.execution.Wid;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
index 80e802cf817ac..8a7eb84157610 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
@@ -212,7 +212,7 @@ public BenchmarkWatcher(Settings settings) {
 
         @Override
         protected TriggerEngine getTriggerEngine(Clock clock, ScheduleRegistry scheduleRegistry) {
-            return new ScheduleTriggerEngineMock(settings, scheduleRegistry, clock);
+            return new ScheduleTriggerEngineMock(scheduleRegistry, clock);
         }
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
index 8efc1d3cf8735..e2b898051abcc 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
index 45b85caacc03a..1a115a4df9587 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
@@ -190,6 +190,6 @@ public void testThatEmptyRequestBodyWorks() throws Exception {
 
     private WatcherSearchTemplateService watcherSearchTemplateService() {
         SearchModule module = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new WatcherSearchTemplateService(Settings.EMPTY, scriptService, new NamedXContentRegistry(module.getNamedXContents()));
+        return new WatcherSearchTemplateService(scriptService, new NamedXContentRegistry(module.getNamedXContents()));
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java
index 4d458e416da09..bd3eccf3b0aaf 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchTransformTests.java
@@ -260,7 +260,7 @@ private SearchTransform.Result executeSearchTransform(WatcherSearchTemplateReque
 
     private WatcherSearchTemplateService watcherSearchTemplateService() {
         String master = internalCluster().getMasterName();
-        return new WatcherSearchTemplateService(internalCluster().clusterService(master).getSettings(),
+        return new WatcherSearchTemplateService(
                 internalCluster().getInstance(ScriptService.class, master),
                 xContentRegistry()
         );
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
index 87c10c97c8fc6..143f760f23c1e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformTests.java
index edf05fac33828..ddc1e07534ef5 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformTests.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.watcher.transform.chain;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -144,7 +144,7 @@ private static class NamedExecutableTransform extends ExecutableTransform<NamedE
         }
 
         NamedExecutableTransform(Transform transform) {
-            super(transform, Loggers.getLogger(NamedExecutableTransform.class));
+            super(transform, LogManager.getLogger(NamedExecutableTransform.class));
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
index e82c868fcc6f4..d874797eca14e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.condition.AlwaysCondition;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java
index 4f0c78b1e0d4a..9ed7d1a3091d2 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/ScheduleTriggerEngineMock.java
@@ -7,7 +7,6 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
@@ -34,8 +33,8 @@ public class ScheduleTriggerEngineMock extends ScheduleTriggerEngine {
 
     private final ConcurrentMap<String, Watch> watches = new ConcurrentHashMap<>();
 
-    public ScheduleTriggerEngineMock(Settings settings, ScheduleRegistry scheduleRegistry, Clock clock) {
-        super(settings, scheduleRegistry, clock);
+    public ScheduleTriggerEngineMock(ScheduleRegistry scheduleRegistry, Clock clock) {
+        super(scheduleRegistry, clock);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/TriggerServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/TriggerServiceTests.java
index f0d3e88b127eb..6372823d36d97 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/TriggerServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/TriggerServiceTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.trigger;
 
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapper;
 import org.elasticsearch.xpack.core.watcher.actions.ExecutableAction;
@@ -39,7 +38,7 @@ public class TriggerServiceTests extends ESTestCase {
     public void setupTriggerService() {
         TriggerEngine triggerEngine = mock(TriggerEngine.class);
         when(triggerEngine.type()).thenReturn(ENGINE_TYPE);
-        service = new TriggerService(Settings.EMPTY, Collections.singleton(triggerEngine));
+        service = new TriggerService(Collections.singleton(triggerEngine));
 
         // simple watch, input and simple action
         watch1 = createWatch("1");
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
index 1d51f390dc5bd..745f1520fe0c6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.watch;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.SearchRequest;
@@ -12,7 +13,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -164,7 +164,7 @@ public void init() throws Exception {
         templateEngine = mock(TextTemplateEngine.class);
         htmlSanitizer = mock(HtmlSanitizer.class);
         licenseState = mock(XPackLicenseState.class);
-        logger = Loggers.getLogger(WatchTests.class);
+        logger = LogManager.getLogger(WatchTests.class);
         searchTemplateService = mock(WatcherSearchTemplateService.class);
     }
 
@@ -177,8 +177,8 @@ public void testParserSelfGenerated() throws Exception {
         Schedule schedule = randomSchedule();
         Trigger trigger = new ScheduleTrigger(schedule);
         ScheduleRegistry scheduleRegistry = registry(schedule);
-        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(Settings.EMPTY, scheduleRegistry, clock);
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, singleton(triggerEngine));
+        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(scheduleRegistry, clock);
+        TriggerService triggerService = new TriggerService(singleton(triggerEngine));
 
         ExecutableInput input = randomInput();
         InputRegistry inputRegistry = registry(input.type());
@@ -205,7 +205,7 @@ public void testParserSelfGenerated() throws Exception {
 
         BytesReference bytes = BytesReference.bytes(jsonBuilder().value(watch));
         logger.info("{}", bytes.utf8ToString());
-        WatchParser watchParser = new WatchParser(settings, triggerService, actionRegistry, inputRegistry, null, clock);
+        WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock);
 
         Watch parsedWatch = watchParser.parse("_name", includeStatus, bytes, XContentType.JSON);
 
@@ -226,7 +226,7 @@ public void testThatBothStatusFieldsCanBeRead() throws Exception {
         InputRegistry inputRegistry = mock(InputRegistry.class);
         ActionRegistry actionRegistry = mock(ActionRegistry.class);
         // a fake trigger service that advances past the trigger end object, which cannot be done with mocking
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, Collections.emptySet()) {
+        TriggerService triggerService = new TriggerService(Collections.emptySet()) {
             @Override
             public Trigger parseTrigger(String jobName, XContentParser parser) throws IOException {
                 XContentParser.Token token;
@@ -248,7 +248,7 @@ public Trigger parseTrigger(String jobName, XContentParser parser) throws IOExce
         }
         WatchStatus watchStatus = new WatchStatus(new DateTime(clock.millis()), unmodifiableMap(actionsStatuses));
 
-        WatchParser watchParser = new WatchParser(settings, triggerService, actionRegistry, inputRegistry, null, clock);
+        WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock);
         XContentBuilder builder = jsonBuilder().startObject().startObject("trigger").endObject().field("status", watchStatus).endObject();
         Watch watch = watchParser.parse("foo", true, BytesReference.bytes(builder), XContentType.JSON);
         assertThat(watch.status().state().getTimestamp().getMillis(), is(clock.millis()));
@@ -260,8 +260,8 @@ public Trigger parseTrigger(String jobName, XContentParser parser) throws IOExce
     public void testParserBadActions() throws Exception {
         ClockMock clock = ClockMock.frozen();
         ScheduleRegistry scheduleRegistry = registry(randomSchedule());
-        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(Settings.EMPTY, scheduleRegistry, clock);
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, singleton(triggerEngine));
+        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(scheduleRegistry, clock);
+        TriggerService triggerService = new TriggerService(singleton(triggerEngine));
         ConditionRegistry conditionRegistry = conditionRegistry();
         ExecutableInput input = randomInput();
         InputRegistry inputRegistry = registry(input.type());
@@ -276,7 +276,7 @@ public void testParserBadActions() throws Exception {
                 .startObject()
                     .startArray("actions").endArray()
                 .endObject();
-        WatchParser watchParser = new WatchParser(settings, triggerService, actionRegistry, inputRegistry, null, clock);
+        WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, clock);
         try {
             watchParser.parse("failure", false, BytesReference.bytes(jsonBuilder), XContentType.JSON);
             fail("This watch should fail to parse as actions is an array");
@@ -288,8 +288,8 @@ public void testParserBadActions() throws Exception {
     public void testParserDefaults() throws Exception {
         Schedule schedule = randomSchedule();
         ScheduleRegistry scheduleRegistry = registry(schedule);
-        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(Settings.EMPTY, scheduleRegistry, Clock.systemUTC());
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, singleton(triggerEngine));
+        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(scheduleRegistry, Clock.systemUTC());
+        TriggerService triggerService = new TriggerService(singleton(triggerEngine));
 
         ConditionRegistry conditionRegistry = conditionRegistry();
         InputRegistry inputRegistry = registry(new ExecutableNoneInput().type());
@@ -302,7 +302,7 @@ public void testParserDefaults() throws Exception {
                 .field(ScheduleTrigger.TYPE, schedule(schedule).build())
                 .endObject();
         builder.endObject();
-        WatchParser watchParser = new WatchParser(settings, triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC());
+        WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC());
         Watch watch = watchParser.parse("failure", false, BytesReference.bytes(builder), XContentType.JSON);
         assertThat(watch, notNullValue());
         assertThat(watch.trigger(), instanceOf(ScheduleTrigger.class));
@@ -316,16 +316,16 @@ public void testParserDefaults() throws Exception {
     public void testParseWatch_verifyScriptLangDefault() throws Exception {
         ScheduleRegistry scheduleRegistry = registry(new IntervalSchedule(new IntervalSchedule.Interval(1,
                 IntervalSchedule.Interval.Unit.SECONDS)));
-        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(Settings.EMPTY, scheduleRegistry, Clock.systemUTC());
-        TriggerService triggerService = new TriggerService(Settings.EMPTY, singleton(triggerEngine));
+        TriggerEngine triggerEngine = new ParseOnlyScheduleTriggerEngine(scheduleRegistry, Clock.systemUTC());
+        TriggerService triggerService = new TriggerService(singleton(triggerEngine));
 
         ConditionRegistry conditionRegistry = conditionRegistry();
         InputRegistry inputRegistry = registry(SearchInput.TYPE);
         TransformRegistry transformRegistry = transformRegistry();
         ActionRegistry actionRegistry = registry(Collections.emptyList(), conditionRegistry, transformRegistry);
-        WatchParser watchParser = new WatchParser(settings, triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC());
+        WatchParser watchParser = new WatchParser(triggerService, actionRegistry, inputRegistry, null, Clock.systemUTC());
 
-        WatcherSearchTemplateService searchTemplateService = new WatcherSearchTemplateService(settings, scriptService, xContentRegistry());
+        WatcherSearchTemplateService searchTemplateService = new WatcherSearchTemplateService(scriptService, xContentRegistry());
 
         XContentBuilder builder = jsonBuilder();
         builder.startObject();
@@ -566,8 +566,8 @@ protected NamedXContentRegistry xContentRegistry() {
 
     public static class ParseOnlyScheduleTriggerEngine extends ScheduleTriggerEngine {
 
-        public ParseOnlyScheduleTriggerEngine(Settings settings, ScheduleRegistry registry, Clock clock) {
-            super(settings, registry, clock);
+        public ParseOnlyScheduleTriggerEngine(ScheduleRegistry registry, Clock clock) {
+            super(registry, clock);
         }
 
         @Override
diff --git a/x-pack/qa/audit-tests/build.gradle b/x-pack/qa/audit-tests/build.gradle
index 8af672fe92aee..126e3834bab4a 100644
--- a/x-pack/qa/audit-tests/build.gradle
+++ b/x-pack/qa/audit-tests/build.gradle
@@ -17,6 +17,7 @@ project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
 
 integTestCluster {
   distribution 'zip'
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle
index ae77c8f893543..7d705d5b0dc9f 100644
--- a/x-pack/qa/build.gradle
+++ b/x-pack/qa/build.gradle
@@ -18,18 +18,3 @@ subprojects {
     }
   }
 }
-
-/* Remove assemble on all qa projects because we don't need to publish
- * artifacts for them. */
-gradle.projectsEvaluated {
-  subprojects {
-    Task assemble = project.tasks.findByName('assemble')
-    if (assemble) {
-      assemble.enabled = false
-    }
-    Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
-    if (dependenciesInfo) {
-      dependenciesInfo.enabled = false
-    }
-  }
-}
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 7f2706a773aa9..e2c2f585a8246 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -23,6 +23,7 @@ integTestRunner {
 }
 
 integTestCluster {
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
index e483645390be4..e2a34f8ed0919 100644
--- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
+++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java
@@ -6,10 +6,10 @@
 
 package org.elasticsearch.xpack.security.authc.kerberos;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
@@ -82,7 +82,7 @@ public abstract class KerberosTestCase extends ESTestCase {
     @BeforeClass
     public static void setupKerberos() throws Exception {
         if (isLocaleUnsupported()) {
-            Logger logger = Loggers.getLogger(KerberosTestCase.class);
+            Logger logger = LogManager.getLogger(KerberosTestCase.class);
             logger.warn("Attempting to run Kerberos test on {} locale, but that breaks SimpleKdcServer. Switching to English.",
                     Locale.getDefault());
             restoreLocale = Locale.getDefault();
diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
index 53bf9e2b78d0b..13601d2fe202f 100644
--- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
+++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
@@ -14,10 +14,10 @@
 import org.apache.kerby.kerberos.kerb.server.KdcConfigKey;
 import org.apache.kerby.kerberos.kerb.server.SimpleKdcServer;
 import org.apache.kerby.util.NetworkUtil;
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
@@ -37,7 +37,7 @@
  * Starts in memory Ldap server and then uses it as backend for Kdc Server.
  */
 public class SimpleKdcLdapServer {
-    private static final Logger logger = Loggers.getLogger(SimpleKdcLdapServer.class);
+    private static final Logger logger = LogManager.getLogger(SimpleKdcLdapServer.class);
 
     private Path workDir = null;
     private SimpleKdcServer simpleKdc;
@@ -221,4 +221,4 @@ public Void run() throws Exception {
         logger.info("SimpleKdcServer stoppped.");
     }
 
-}
\ No newline at end of file
+}
diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle
index b2f9c17270c87..bfa0f042627cc 100644
--- a/x-pack/qa/full-cluster-restart/build.gradle
+++ b/x-pack/qa/full-cluster-restart/build.gradle
@@ -147,7 +147,12 @@ subprojects {
     configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
       dependsOn copyTestNodeKeystore
       if (version.before('6.3.0')) {
-        mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
+          String depVersion = version;
+          if (project.bwcVersions.unreleased.contains(version)) {
+              depVersion += "-SNAPSHOT"
+          }
+          mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
+
       }
       bwcVersion = version
       numBwcNodes = 2
@@ -249,7 +254,7 @@ subprojects {
   // basic integ tests includes testing bwc against the most recent version
   task integTest {
     if (project.bwc_tests_enabled) {
-      for (final def version : bwcVersions.snapshotsIndexCompatible) {
+      for (final def version : bwcVersions.unreleasedIndexCompatible) {
         dependsOn "v${version}#bwcTest"
       }
     }
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index 762eae6d9e032..3b576ec537f5a 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
 import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle
index e79490df829e3..c31b2c0ad1d5e 100644
--- a/x-pack/qa/multi-cluster-search-security/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/build.gradle
@@ -16,6 +16,7 @@ remoteClusterTestCluster {
   numNodes = 2
   clusterName = 'remote-cluster'
   setting 'cluster.remote.connect', false
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle
index 4369287caba32..243a6f40438cc 100644
--- a/x-pack/qa/multi-node/build.gradle
+++ b/x-pack/qa/multi-node/build.gradle
@@ -8,6 +8,7 @@ dependencies {
 integTestCluster {
   numNodes = 2
   clusterName = 'multi-node'
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
diff --git a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
index 3ea1b8e67471c..d4f35c5099064 100644
--- a/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
+++ b/x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 
 import java.io.IOException;
 import java.time.Instant;
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index 97c0e8e17fee7..ea2b7d6990622 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -12,6 +12,7 @@ dependencies {
 integTestCluster {
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml
index 407a1c1849526..a5779ff94d06d 100644
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/10_reindex.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Reindex as same user works":
 
diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml
index 3c31b8cc5b039..b558ad7267718 100644
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/15_reindex_from_remote.yml
@@ -163,6 +163,8 @@
 
 ---
 "Using a script to write to an index to which you don't have access is forbidden even if you read as a superuser":
+  - skip:
+      features: headers
   - do:
       index:
         index:        source
diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml
index 627b29ea8b529..caf29f2fc6b6c 100644
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/20_update_by_query.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Update_by_query as same user works":
 
diff --git a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml
index 59b6f2b7792a6..deffc4ce5e2c5 100644
--- a/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml
+++ b/x-pack/qa/reindex-tests-with-security/src/test/resources/rest-api-spec/test/30_delete_by_query.yml
@@ -1,3 +1,6 @@
+setup:
+  - skip:
+      features: headers
 ---
 "Delete_by_query as same user works":
 
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index b25d81648def8..b9bf8b8606dc4 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -120,7 +120,11 @@ subprojects {
     configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
       dependsOn copyTestNodeKeystore
       if (version.before('6.3.0')) {
-        mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
+          String depVersion = version;
+          if (project.bwcVersions.unreleased.contains(version)) {
+              depVersion += "-SNAPSHOT"
+          }
+          mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
       }
       String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
       setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
@@ -319,7 +323,7 @@ subprojects {
   // basic integ tests includes testing bwc against the most recent version
   task integTest {
     if (project.bwc_tests_enabled) {
-      for (final def version : bwcVersions.snapshotsWireCompatible) {
+      for (final def version : bwcVersions.unreleasedWireCompatible) {
         dependsOn "v${version}#bwcTest"
       }
     }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
index a1430965339c3..3b72674ed0751 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
@@ -43,6 +43,11 @@ protected boolean preserveRollupJobsUponCompletion() {
         return true;
     }
 
+    @Override
+    protected boolean preserveILMPoliciesUponCompletion() {
+        return true;
+    }
+
     enum ClusterType {
         OLD,
         MIXED,
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java
index 0da79b27bfaf6..f51835af7196b 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.xpack.core.watcher.support.xcontent.ObjectPath;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
index 12add3d7feaa8..5a9c866058dc2 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -64,6 +64,11 @@ protected boolean preserveRollupJobsUponCompletion() {
         return true;
     }
 
+    @Override
+    protected boolean preserveILMPoliciesUponCompletion() {
+        return true;
+    }
+
     public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml
index 119f6f4874960..7f29b31ab1fe1 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_security.yml
@@ -1,5 +1,7 @@
 ---
 "Verify native store security actions":
+  - skip:
+      features: headers
   # create native user and role
   - do:
      xpack.security.put_user:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml
index 46ade4823a221..a265605c536d2 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_security.yml
@@ -1,5 +1,7 @@
 ---
 "Verify user and role in upgraded cluster":
+  - skip:
+      features: headers
   - do:
       headers:
         Authorization: "Basic bmF0aXZlX3VzZXI6eC1wYWNrLXRlc3QtcGFzc3dvcmQ="
diff --git a/x-pack/qa/security-client-tests/build.gradle b/x-pack/qa/security-client-tests/build.gradle
index e676e55a152d4..556e36e51467f 100644
--- a/x-pack/qa/security-client-tests/build.gradle
+++ b/x-pack/qa/security-client-tests/build.gradle
@@ -19,6 +19,7 @@ integTestRunner {
 }
 
 integTestCluster {
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle
index aef4fc33f6abe..c75573431fb50 100644
--- a/x-pack/qa/security-example-spi-extension/build.gradle
+++ b/x-pack/qa/security-example-spi-extension/build.gradle
@@ -27,6 +27,7 @@ integTestCluster {
   setting 'xpack.security.authc.realms.native.type', 'native'
   setting 'xpack.security.authc.realms.native.order', '2'
   setting 'xpack.security.enabled', 'true'
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
index 90b5eefcb56d4..3f85d8086d678 100644
--- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
+++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java
@@ -56,11 +56,11 @@ public AuthenticationFailureHandler getAuthenticationFailureHandler() {
     @Override
     public List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>>
     getRolesProviders(Settings settings, ResourceWatcherService resourceWatcherService) {
-        CustomInMemoryRolesProvider rp1 = new CustomInMemoryRolesProvider(settings, Collections.singletonMap(ROLE_A, "read"));
+        CustomInMemoryRolesProvider rp1 = new CustomInMemoryRolesProvider(Collections.singletonMap(ROLE_A, "read"));
         Map<String, String> roles = new HashMap<>();
         roles.put(ROLE_A, "all");
         roles.put(ROLE_B, "all");
-        CustomInMemoryRolesProvider rp2 = new CustomInMemoryRolesProvider(settings, roles);
+        CustomInMemoryRolesProvider rp2 = new CustomInMemoryRolesProvider(roles);
         return Arrays.asList(rp1, rp2);
     }
 }
diff --git a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/role/CustomInMemoryRolesProvider.java b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/role/CustomInMemoryRolesProvider.java
index 0d5a71e6244b4..44616054dc7a8 100644
--- a/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/role/CustomInMemoryRolesProvider.java
+++ b/x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/role/CustomInMemoryRolesProvider.java
@@ -7,7 +7,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 
@@ -30,8 +29,7 @@ public class CustomInMemoryRolesProvider
 
     private final Map<String, String> rolePermissionSettings;
 
-    public CustomInMemoryRolesProvider(Settings settings, Map<String, String> rolePermissionSettings) {
-        super(settings);
+    public CustomInMemoryRolesProvider(Map<String, String> rolePermissionSettings) {
         this.rolePermissionSettings = rolePermissionSettings;
     }
 
diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
index 7813ff3d3d56c..57be337f634f2 100644
--- a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
@@ -8,6 +8,7 @@ dependencies {
 }
 
 integTestCluster {
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'true'
   setting 'xpack.watcher.enabled', 'true'
   setting 'xpack.security.enabled', 'false'
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index 4f338d07fb531..76c887ab04a2b 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -65,12 +65,14 @@ integTestCluster {
   setting 'xpack.monitoring.exporters._http.ssl.verification_mode', 'full'
 
   setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.http.ssl.enabled', 'true'
   setting 'xpack.security.http.ssl.key', 'testnode.pem'
   setting 'xpack.security.http.ssl.certificate', 'testnode.crt'
   keystoreSetting 'xpack.security.http.ssl.secure_key_passphrase', 'testnode'
 
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   // copy keystores, keys and certificates into config/
   extraConfigFile nodeKeystore.name, nodeKeystore
diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
index 50e217b28b270..f5007e5b0910b 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
@@ -14,6 +14,7 @@ task copyWatcherRestTests(type: Copy) {
 
 integTestCluster {
   dependsOn copyWatcherRestTests
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml
index 0b74ebb0c0058..e3c512560a992 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/10_insufficient_privs.yml
@@ -1,5 +1,7 @@
 ---
 "Test watcher is protected by security":
+  - skip:
+      features: headers
   - do:
       headers: { es-security-runas-user: powerless_user }
       catch: forbidden
diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
index 7a0634f5187b1..779b41748acf0 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/resources/rest-api-spec/test/watcher/watcher_and_security/20_test_run_as_execute_watch.yml
@@ -28,6 +28,8 @@ teardown:
 
 ---
 "Test watch search input is run as user who added the watch":
+  - skip:
+      features: headers
   - do:
       xpack.watcher.put_watch:
         id: "my_watch"
@@ -81,6 +83,8 @@ teardown:
 
 ---
 "Test watch is runas user properly recorded":
+  - skip:
+      features: headers
   - do:
       xpack.watcher.put_watch:
         id: "my_watch"
@@ -133,7 +137,8 @@ teardown:
 
 ---
 "Test watch search input does not work against index user is not allowed to read":
-
+  - skip:
+      features: headers
   - do:
       # by impersonating this request as powerless user we cannot query the my_test_index
       # headers: { es-security-runas-user: powerless_user }
@@ -290,6 +295,8 @@ teardown:
 
 ---
 "Test watch index action requires permission to write to an index":
+  - skip:
+      features: headers
   - do:
       xpack.watcher.put_watch:
         id: "my_watch"
@@ -339,6 +346,8 @@ teardown:
 
 ---
 "Test watch index action does not work without permissions":
+  - skip:
+      features: headers
   - do:
       xpack.watcher.put_watch:
         id: "my_watch"
diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle
index 5923afcacad94..fc22fe9aa065f 100644
--- a/x-pack/qa/smoke-test-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-watcher/build.gradle
@@ -14,6 +14,7 @@ dependencies {
 }
 
 integTestCluster {
+  setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java
index feeea24871240..e9c5106d44e87 100644
--- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java
+++ b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/WatcherTemplateIT.java
@@ -41,7 +41,7 @@ public void init() throws Exception {
         Map<String, ScriptContext<?>> contexts =
             Collections.singletonMap(Watcher.SCRIPT_TEMPLATE_CONTEXT.name, Watcher.SCRIPT_TEMPLATE_CONTEXT);
         ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, contexts);
-        textTemplateEngine = new TextTemplateEngine(Settings.EMPTY, scriptService);
+        textTemplateEngine = new TextTemplateEngine(scriptService);
     }
 
     public void testEscaping() throws Exception {
diff --git a/x-pack/qa/sql/src/main/resources/nested.csv-spec b/x-pack/qa/sql/src/main/resources/nested.csv-spec
deleted file mode 100644
index 428ed78120440..0000000000000
--- a/x-pack/qa/sql/src/main/resources/nested.csv-spec
+++ /dev/null
@@ -1,112 +0,0 @@
-//
-//  Nested documents
-//
-// CsvJdbc has issues with foo.bar so msot fields are aliases or wrapped inside a function
-
-describeParent
-DESCRIBE test_emp;
-
-       column       |     type      |    mapping    
---------------------+---------------+---------------
-birth_date          |TIMESTAMP      |DATE           
-dep                 |STRUCT         |NESTED         
-dep.dep_id          |VARCHAR        |KEYWORD        
-dep.dep_name        |VARCHAR        |TEXT           
-dep.dep_name.keyword|VARCHAR        |KEYWORD        
-dep.from_date       |TIMESTAMP      |DATE           
-dep.to_date         |TIMESTAMP      |DATE           
-emp_no              |INTEGER        |INTEGER        
-first_name          |VARCHAR        |TEXT           
-first_name.keyword  |VARCHAR        |KEYWORD        
-gender              |VARCHAR        |KEYWORD        
-hire_date           |TIMESTAMP      |DATE           
-languages           |TINYINT        |BYTE           
-last_name           |VARCHAR        |TEXT           
-last_name.keyword   |VARCHAR        |KEYWORD        
-salary              |INTEGER        |INTEGER   
-;
-
-// disable until we figure out how to use field names with . in their name
-//nestedStar
-//SELECT dep.* FROM test_emp ORDER BY dep.dep_id LIMIT 5;
-
-//dep.dep_id:s         | dep.dep_name:s       | dep.from_date:ts     | dep.to_date:ts
-
-//d001                 | Marketing            | 744336000000         | 253370764800000
-//d001                 | Marketing            | 704332800000         | 806371200000
-//d001                 | Marketing            | 577929600000         | 253370764800000
-//d002                 | Finance              | 732672000000         | 965865600000
-//d007                 | Sales                | 720921600000         | 253370764800000
-//;
-
-filterPerNestedWithOrderByTopLevel
-SELECT first_name f, last_name l, YEAR(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY f LIMIT 5;
-
-f:s                  | l:s                  | d:i
-
-Chirstian            |Koblick               |1986           
-Duangkaew            |Piveteau              |1996           
-Gino                 |Leonhardt             |1989           
-Hidefumi             |Caine                 |1992           
-Jayson               |Mandell               |1999    
-;
-
-filterPerNestedWithOrderByNested
-SELECT first_name f, last_name l, YEAR(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY dep.from_date LIMIT 5;
-
-f:s                  | l:s                  | d:i
-
-Sreekrishna          |Servieres             |1985           
-Zhongwei             |Rosen                 |1986           
-Chirstian            |Koblick               |1986           
-Vishv                |Zockler               |1987           
-null                 |Chappelet             |1988    
-;
-
-filterPerNestedWithOrderByNestedWithAlias
-SELECT first_name f, dep.dep_id i, MONTH(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY i LIMIT 5;
-
-f:s                  | i:s                  | d:i
-
-Parto                | d004                 | 12
-Chirstian            | d004                 | 12
-Duangkaew            | d004                 | 11
-Kazuhide             | d004                 | 7
-Mayuko               | d004                 | 12
-;
-
-filterPerNestedWithOrderByNestedWithoutProjection
-SELECT first_name f, MONTH(dep.from_date) d FROM test_emp WHERE dep.dep_name = 'Production' ORDER BY dep.dep_id LIMIT 5;
-
-f:s        | d:i
-
-Parto      | 12
-Chirstian  | 12
-Duangkaew  | 11
-Kazuhide   | 7
-Mayuko     | 12
-;
-
-selectWithScalarOnNested
-SELECT first_name f, last_name l, YEAR(dep.from_date) start FROM test_emp WHERE dep.dep_name = 'Production' AND languages > 1 ORDER BY start LIMIT 5;
-
-f:s                  | l:s                  | start:i
-
-Sreekrishna          |Servieres      |1985           
-Zhongwei             |Rosen          |1986           
-Chirstian            |Koblick        |1986           
-null                 |Chappelet      |1988           
-Zvonko               |Nyanchama      |1989     
-;
-
-selectWithScalarOnNestedWithoutProjection
-SELECT first_name f, last_name l FROM test_emp WHERE dep.dep_name = 'Production' AND languages > 1 ORDER BY YEAR(dep.from_date) LIMIT 5;
-
-f:s                  | l:s
-
-Sreekrishna          |Servieres      
-Zhongwei             |Rosen          
-Chirstian            |Koblick        
-null                 |Chappelet      
-Zvonko               |Nyanchama  
-;
diff --git a/x-pack/qa/sql/src/main/resources/nulls.csv-spec b/x-pack/qa/sql/src/main/resources/nulls.csv-spec
deleted file mode 100644
index 1cb9a1ed7f319..0000000000000
--- a/x-pack/qa/sql/src/main/resources/nulls.csv-spec
+++ /dev/null
@@ -1,25 +0,0 @@
-//
-// Null expressions
-//
-
-nullDate
-SELECT YEAR(CAST(NULL AS DATE)) d;
-
-d:i
-null
-;
-
-nullAdd
-SELECT CAST(NULL AS INT) + CAST(NULL AS FLOAT) AS n;
-
-n:d
-null
-;
-
-
-nullDiv
-SELECT 5 / CAST(NULL AS FLOAT) + 10 AS n;
-
-n:d
-null
-;
diff --git a/x-pack/qa/sql/src/main/resources/select.csv-spec b/x-pack/qa/sql/src/main/resources/select.csv-spec
deleted file mode 100644
index bf208c62026df..0000000000000
--- a/x-pack/qa/sql/src/main/resources/select.csv-spec
+++ /dev/null
@@ -1,102 +0,0 @@
-// SELECT with IN
-inWithLiterals
-SELECT 1 IN (1, 2, 3), 1 IN (2, 3);
-
-  1 IN (1, 2, 3) |  1 IN (2, 3)
------------------+-------------
-true             |false
-;
-
-inWithLiteralsAndFunctions
-SELECT 1 IN (2 - 1, 2, 3), abs(-1) IN (2, 3, abs(4 - 5));
-
-  1 IN (1, 2, 3) |  1 IN (2, 3)
------------------+-------------
-true             |false
-;
-
-
-inWithLiteralsAndNegation
-SELECT NOT 1 IN (1, 1 + 1, 3), NOT 1 IN (2, 3);
-
-  1 IN (1, 2, 3) |  1 IN (2, 3)
------------------+-------------
-false            |true
-;
-
-
-inWithNullHandling
-SELECT 2 IN (1, null, 3), 3 IN (1, null, 3), null IN (1, null, 3), null IN (1, 2, 3);
-
-  2 IN (1, null, 3) |  3 IN (1, null, 3) |  null IN (1, null, 3) |  null IN (1, 2, 3)
---------------------+--------------------+-----------------------+-------------------
-null                |true                |null                   | null
-;
-
-inWithNullHandlingAndNegation
-SELECT NOT 2 IN (1, null, 3), NOT 3 IN (1, null, 3), NOT null IN (1, null, 3), NOT null IN (1, 2, 3);
-
-  NOT 2 IN (1, null, 3) |  NOT 3 IN (1, null, 3) |  NOT null IN (1, null, 3) |  null IN (1, 2, 3)
-------------------------+------------------------+---------------------------+--------------------
-null                    |false                   |null                       | null
-;
-
-//
-// SELECT with IN and table columns
-//
-inWithTableColumn
-SELECT emp_no IN (10000, 10001, 10002) FROM test_emp ORDER BY 1;
-
- emp_no
--------
-10001
-10002
-;
-
-inWithTableColumnAndFunction
-SELECT emp_no IN (10000, 10000 + 1, abs(-10000 - 2)) FROM test_emp;
-
- emp_no
--------
-10001
-10002
-;
-
-inWithTableColumnAndNegation
-SELECT emp_no NOT IN (10000, 10000 + 1, 10002) FROM test_emp ORDER BY 1 LIMIT 3;
-
- emp_no
--------
-10003
-10004
-10005
-;
-
-inWithTableColumnAndComplexFunctions
-SELECT 1 IN (1, abs(2 - 4), 3) OR emp_no NOT IN (10000, 10000 + 1, 10002) FROM test_emp ORDER BY 1 LIMIT 3;
-
- emp_no
--------
-10003
-10004
-10005
-;
-
-inWithTableColumnAndNullHandling
-SELECT emp_no, birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)), birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), null, CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)) FROM test_emp WHERE emp_no = 10038 OR emp_no = 10039 OR emp_no = 10040 ORDER BY 1;
-
- emp_no |  birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)) |  birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), null, CAST('1959-10-01T00:00:00Z' AS TIMESTAMP))
---------+-------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------
-10038   | true                                                                                                  | true
-10039   | null                                                                                                  | null
-10040   | false                                                                                                 | null
-
-
-inWithTableColumnAndNullHandlingAndNegation
-SELECT emp_no, NOT birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)), NOT birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), null, CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)) FROM test_emp WHERE emp_no = 10038 OR emp_no = 10039 OR emp_no = 10040 ORDER BY 1;
-
- emp_no |  NOT birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), CAST('1959-10-01T00:00:00Z' AS TIMESTAMP)) |  NOT birth_date in (CAST('2018-10-01T00:00:00Z' AS TIMESTAMP), null, CAST('1959-10-01T00:00:00Z' AS TIMESTAMP))
---------+-----------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------
-10038   | false                                                                                                     | false
-10039   | null                                                                                                      | null
-10040   | true                                                                                                      | null
\ No newline at end of file