Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (30 commits)
  Add wait_for_no_initializing_shards to cluster health API (#27489)
  Carry over version map size to prevent excessive resizing (#27516)
  Fix scroll query with a sort that is a prefix of the index sort (#27498)
  Delete shard store files before restoring a snapshot (#27476)
  CURRENT should not be a -SNAPSHOT version if build.snapshot is false (#27512)
  Fix merging of _meta field (#27352)
  test: do not run percolator query builder bwc test against 5.x versions
  Remove unused method (#27508)
  Consolidate version numbering semantics (#27397)
  Adjust CombinedDeletionPolicy for multiple commits (#27456)
  Minor ShapeBuilder cleanup
  [GEO] Deprecate ShapeBuilders and decouple geojson parse logic
  Improve docs for split API in 6.1/6.x (#27504)
  test: use correct pre 6.0.0-alpha1 format
  Update composite-aggregation.asciidoc
  Deprecate `levenstein` in favor of `levenshtein` (#27409)
  Decouple nio constructs from the tcp transport (#27484)
  Bump version from 6.1 to 6.2
  Fix whitespace in Security.java
  Tighten which classes can exit
  ...

build.gradle

@@ -17,15 +17,15 @@
  * under the License.
  */
 
-import java.nio.file.Path
-import java.util.regex.Matcher
-import org.eclipse.jgit.lib.Repository
-import org.eclipse.jgit.lib.RepositoryBuilder
-import org.gradle.plugins.ide.eclipse.model.SourceFolder
+
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
+import org.elasticsearch.gradle.VersionProperties
+import org.gradle.plugins.ide.eclipse.model.SourceFolder
+
+import java.nio.file.Path
 
 // common maven publishing configuration
 subprojects {
@@ -67,61 +67,16 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
   }
 }
 
-/* Introspect all versions of ES that may be tested agains for backwards
+/* Introspect all versions of ES that may be tested against for backwards
  * compatibility. It is *super* important that this logic is the same as the
- * logic in VersionUtils.java, modulo alphas, betas, and rcs which are ignored
- * in gradle because they don't have any backwards compatibility guarantees
- * but are not ignored in VersionUtils.java because the tests expect them not
- * to be. */
-Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
-int prevMajor = currentVersion.major - 1
-File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
-List<String> versionLines = versionFile.readLines('UTF-8')
-List<Version> versions = []
-// keep track of the previous major version's last minor, so we know where wire compat begins
-int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
-int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
-int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
-for (String line : versionLines) {
-  /* Note that this skips alphas and betas which is fine because they aren't
-   * compatible with anything. */
-  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+) .*/
-  if (match.matches()) {
-    int major = Integer.parseInt(match.group(1))
-    int minor = Integer.parseInt(match.group(2))
-    int bugfix = Integer.parseInt(match.group(3))
-    Version foundVersion = new Version(major, minor, bugfix, false)
-    if (currentVersion != foundVersion) {
-      versions.add(foundVersion)
-    }
-    if (major == prevMajor && minor > lastPrevMinor) {
-      prevMinorIndex = versions.size() - 1
-      lastPrevMinor = minor
-    }
-    if (major == prevMajor) {
-      prevBugfixIndex = versions.size() - 1
-    }
-  }
-}
-if (versions.toSorted { it.id } != versions) {
-  println "Versions: ${versions}"
-  throw new GradleException("Versions.java contains out of order version constants")
-}
-if (prevBugfixIndex != -1) {
-  versions[prevBugfixIndex] = new Version(
-      versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor, versions[prevBugfixIndex].bugfix, true)
-}
-if (currentVersion.bugfix == 0) {
-  // If on a release branch, after the initial release of that branch, the bugfix version will
-  // be bumped, and will be != 0. On master and N.x branches, we want to test against the
-  // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
-  // and the bwc distribution will checkout and build that version.
-  Version last = versions[-1]
-  versions[-1] = new Version(last.major, last.minor, last.bugfix, true)
-  if (last.bugfix == 0) {
-    versions[-2] = new Version(
-        versions[-2].major, versions[-2].minor, versions[-2].bugfix, true)
-  }
+ * logic in VersionUtils.java, throwing out alphas because they don't have any
+ * backwards compatibility guarantees and only keeping the latest beta or rc
+ * in a branch if there are only betas and rcs in the branch so we have
+ * *something* to test against. */
+VersionCollection versions = new VersionCollection(file('core/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
+if (versions.currentVersion.toString() != VersionProperties.elasticsearch) {
+  throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " +
+          "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]")
 }
 
 // build metadata from previous build, contains eg hashes for bwc builds
@@ -140,9 +95,10 @@ allprojects {
     // for ide hacks...
     isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
-    // for backcompat testing
-    indexCompatVersions = versions
-    wireCompatVersions = versions.subList(prevMinorIndex, versions.size())
+
+    // for BWC testing
+    versionCollection = versions
+
     buildMetadata = buildMetadataMap
   }
 }
@@ -160,13 +116,13 @@ task verifyVersions {
     Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= prevMajor && it.before(VersionProperties.elasticsearch) }
+    knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
 
     /* Limit the listed versions to those that have been marked as released.
      * Versions not marked as released don't get the same testing and we want
      * to make sure that we flip all unreleased versions to released as soon
      * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(indexCompatVersions.findAll { false == it.snapshot })
+    Set<Version> actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot })
 
     // Finally, compare!
     if (knownVersions.equals(actualVersions) == false) {
@@ -240,30 +196,18 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
-    "org.elasticsearch.plugin:aggs-composite-client:${version}": ':modules:aggs-composite',
   ]
-  if (indexCompatVersions[-1].snapshot) {
-    /* The last and second to last versions can be snapshots. Rather than use
-     * snapshots built by CI we connect these versions to projects that build
-     * those those versions from the HEAD of the appropriate branch. */
-    if (indexCompatVersions[-1].bugfix == 0) {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-    } else {
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
+
+  for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+    if (version.branch != null) {
+      final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}"
+      project(snapshotProject).ext.bwcVersion = version
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject
     }
-  } else if (indexCompatVersions[-2].snapshot) {
-    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
-    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
+
   project.afterEvaluate {
     configurations.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -20,6 +20,8 @@
 package org.elasticsearch.gradle
 
 import groovy.transform.Sortable
+import java.util.regex.Matcher
+import org.gradle.api.InvalidUserDataException
 
 /**
  * Encapsulates comparison and printing logic for an x.y.z version.
@@ -29,35 +31,42 @@ public class Version {
 
     final int major
     final int minor
-    final int bugfix
+    final int revision
     final int id
     final boolean snapshot
+    final String branch
+    /**
+     * Suffix on the version name. Unlike Version.java the build does not
+     * consider alphas and betas different versions, it just preserves the
+     * suffix that the version was declared with in Version.java.
+     */
+    final String suffix
 
-    public Version(int major, int minor, int bugfix, boolean snapshot) {
+    public Version(int major, int minor, int revision,
+            String suffix, boolean snapshot, String branch) {
         this.major = major
         this.minor = minor
-        this.bugfix = bugfix
+        this.revision = revision
         this.snapshot = snapshot
-        this.id = major * 100000 + minor * 1000 + bugfix * 10 +
+        this.suffix = suffix
+        this.branch = branch
+        this.id = major * 100000 + minor * 1000 + revision * 10 +
             (snapshot ? 1 : 0)
     }
 
     public static Version fromString(String s) {
-        String[] parts = s.split('\\.')
-        String bugfix = parts[2]
-        boolean snapshot = false
-        if (bugfix.contains('-')) {
-            snapshot = bugfix.endsWith('-SNAPSHOT')
-            bugfix = bugfix.split('-')[0]
+        Matcher m = s =~ /(\d+)\.(\d+)\.(\d+)(-alpha\d+|-beta\d+|-rc\d+)?(-SNAPSHOT)?/
+        if (m.matches() == false) {
+            throw new InvalidUserDataException("Invalid version [${s}]")
         }
-        return new Version(parts[0] as int, parts[1] as int, bugfix as int,
-            snapshot)
+        return new Version(m.group(1) as int, m.group(2) as int,
+            m.group(3) as int, m.group(4) ?: '', m.group(5) != null, null)
     }
 
     @Override
     public String toString() {
         String snapshotStr = snapshot ? '-SNAPSHOT' : ''
-        return "${major}.${minor}.${bugfix}${snapshotStr}"
+        return "${major}.${minor}.${revision}${suffix}${snapshotStr}"
     }
 
     public boolean before(String compareTo) {
@@ -75,4 +84,43 @@ public class Version {
     public boolean after(String compareTo) {
         return id > fromString(compareTo).id
     }
+
+    public boolean onOrBeforeIncludingSuffix(Version otherVersion) {
+        if (id != otherVersion.id) {
+            return id < otherVersion.id
+        }
+
+        if (suffix == '') {
+            return otherVersion.suffix == ''
+        }
+
+        return otherVersion.suffix == '' || suffix < otherVersion.suffix
+    }
+
+    boolean equals(o) {
+        if (this.is(o)) return true
+        if (getClass() != o.class) return false
+
+        Version version = (Version) o
+
+        if (id != version.id) return false
+        if (major != version.major) return false
+        if (minor != version.minor) return false
+        if (revision != version.revision) return false
+        if (snapshot != version.snapshot) return false
+        if (suffix != version.suffix) return false
+
+        return true
+    }
+
+    int hashCode() {
+        int result
+        result = major
+        result = 31 * result + minor
+        result = 31 * result + revision
+        result = 31 * result + id
+        result = 31 * result + (snapshot ? 1 : 0)
+        result = 31 * result + (suffix != null ? suffix.hashCode() : 0)
+        return result
+    }
 }