Merge remote-tracking branch 'elastic/master' into use-plugins-for-no…

…de-tool
elastic · Dec 4, 2019 · 2fbb88c · 2fbb88c
2 parents ce68d75 + d9560e5
commit 2fbb88c
Show file tree

Hide file tree

Showing 205 changed files with 4,209 additions and 1,292 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -11,7 +11,7 @@ BWC_VERSION:
   - "7.4.0"
   - "7.4.1"
   - "7.4.2"
-  - "7.4.3"
   - "7.5.0"
+  - "7.5.1"
   - "7.6.0"
   - "8.0.0"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -115,6 +115,10 @@ We support development in the Eclipse and IntelliJ IDEs.
 For Eclipse, the minimum version that we support is [4.13][eclipse].
 For IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
 
+[Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with:
+
+    ./gradlew :run
+
 ### Configuring IDEs And Running Tests
 
 Eclipse users can automatically configure their IDE: `./gradlew eclipse`
@@ -153,6 +157,10 @@ For IntelliJ, go to
 For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 `VM Arguments`.
 
+Some tests related to locale testing also require the flag 
+`-Djava.locale.providers` to be set. Set the VM options/VM arguments for
+IntelliJ or Eclipse like describe above to use 
+`-Djava.locale.providers=SPI,COMPAT`.
 
 ### Java Language Formatting Guidelines
 

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
@@ -263,6 +263,10 @@ If you want to just run the precommit checks:
 
 Some of these checks will require `docker-compose` installed for bringing up
 test fixtures. If it's not present those checks will be skipped automatically.
+The host running Docker (or VM if you're using Docker Desktop) needs 4GB of
+memory or some of the containers will fail to start. You can tell that you
+are short of memory if containers are exiting quickly after starting with
+code 137 (128 + 9, where 9 means SIGKILL).
 
 == Testing the REST layer
 

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -33,9 +33,6 @@
 import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
-import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
-import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
-import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
@@ -48,8 +45,11 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.DeleteTrainedModelRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
+import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FindFileStructureResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -135,8 +135,6 @@
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification;
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric;
-import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric.ActualClass;
-import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric.PredictedClass;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression;
@@ -1822,9 +1820,12 @@ public void testEvaluateDataFrame_Classification() throws IOException {
                 accuracyResult.getActualClasses(),
                 equalTo(
                     List.of(
-                        new AccuracyMetric.ActualClass("cat", 5, 0.6),  // 3 out of 5 examples labeled as "cat" were classified correctly
-                        new AccuracyMetric.ActualClass("dog", 4, 0.75),  // 3 out of 4 examples labeled as "dog" were classified correctly
-                        new AccuracyMetric.ActualClass("ant", 1, 0.0))));  // no examples labeled as "ant" were classified correctly
+                        // 3 out of 5 examples labeled as "cat" were classified correctly
+                        new AccuracyMetric.ActualClass("cat", 5, 0.6),
+                        // 3 out of 4 examples labeled as "dog" were classified correctly
+                        new AccuracyMetric.ActualClass("dog", 4, 0.75),
+                        // no examples labeled as "ant" were classified correctly
+                        new AccuracyMetric.ActualClass("ant", 1, 0.0))));
             assertThat(accuracyResult.getOverallAccuracy(), equalTo(0.6));  // 6 out of 10 examples were classified correctly
         }
         {  // No size provided for MulticlassConfusionMatrixMetric, default used instead
@@ -1846,20 +1847,29 @@ public void testEvaluateDataFrame_Classification() throws IOException {
                 mcmResult.getConfusionMatrix(),
                 equalTo(
                     List.of(
-                        new ActualClass(
+                        new MulticlassConfusionMatrixMetric.ActualClass(
                             "ant",
                             1L,
-                            List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 1L), new PredictedClass("dog", 0L)),
+                            List.of(
+                                new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("dog", 0L)),
                             0L),
-                        new ActualClass(
+                        new MulticlassConfusionMatrixMetric.ActualClass(
                             "cat",
                             5L,
-                            List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 3L), new PredictedClass("dog", 1L)),
+                            List.of(
+                                new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("cat", 3L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("dog", 1L)),
                             1L),
-                        new ActualClass(
+                        new MulticlassConfusionMatrixMetric.ActualClass(
                             "dog",
                             4L,
-                            List.of(new PredictedClass("ant", 0L), new PredictedClass("cat", 1L), new PredictedClass("dog", 3L)),
+                            List.of(
+                                new MulticlassConfusionMatrixMetric.PredictedClass("ant", 0L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("dog", 3L)),
                             0L))));
             assertThat(mcmResult.getOtherActualClassCount(), equalTo(0L));
         }
@@ -1882,8 +1892,20 @@ public void testEvaluateDataFrame_Classification() throws IOException {
                 mcmResult.getConfusionMatrix(),
                 equalTo(
                     List.of(
-                        new ActualClass("cat", 5L, List.of(new PredictedClass("cat", 3L), new PredictedClass("dog", 1L)), 1L),
-                        new ActualClass("dog", 4L, List.of(new PredictedClass("cat", 1L), new PredictedClass("dog", 3L)), 0L)
+                        new MulticlassConfusionMatrixMetric.ActualClass(
+                            "cat",
+                            5L,
+                            List.of(
+                                new MulticlassConfusionMatrixMetric.PredictedClass("cat", 3L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("dog", 1L)),
+                            1L),
+                        new MulticlassConfusionMatrixMetric.ActualClass(
+                            "dog",
+                            4L,
+                            List.of(
+                                new MulticlassConfusionMatrixMetric.PredictedClass("cat", 1L),
+                                new MulticlassConfusionMatrixMetric.PredictedClass("dog", 3L)),
+                            0L)
                     )));
             assertThat(mcmResult.getOtherActualClassCount(), equalTo(1L));
         }

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -765,6 +765,7 @@ public void testApiNamingConventions() throws Exception {
             "cluster.remote_info",
             "create",
             "get_script_context",
+            "get_script_languages",
             "get_source",
             "indices.exists_type",
             "indices.get_upgrade",

diff --git a/...-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/...-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -83,7 +83,6 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.TaskId;
 
 import java.util.Collections;
@@ -833,10 +832,6 @@ public void testReindex() throws Exception {
             // tag::reindex-request-pipeline
             request.setDestPipeline("my_pipeline"); // <1>
             // end::reindex-request-pipeline
-            // tag::reindex-request-sort
-            request.addSortField("field1", SortOrder.DESC); // <1>
-            request.addSortField("field2", SortOrder.ASC); // <2>
-            // end::reindex-request-sort
             // tag::reindex-request-script
             request.setScript(
                 new Script(

diff --git a/...-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java b/...-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java
@@ -20,36 +20,56 @@
 
 import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification;
+import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetricResultTests;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetricResultTests;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetricResultTests;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.Predicate;
 
 public class EvaluateDataFrameResponseTests extends AbstractXContentTestCase<EvaluateDataFrameResponse> {
 
     public static EvaluateDataFrameResponse randomResponse() {
-        List<EvaluationMetric.Result> metrics = new ArrayList<>();
-        if (randomBoolean()) {
-            metrics.add(AucRocMetricResultTests.randomResult());
+        String evaluationName = randomFrom(BinarySoftClassification.NAME, Classification.NAME, Regression.NAME);
+        List<EvaluationMetric.Result> metrics;
+        switch (evaluationName) {
+            case BinarySoftClassification.NAME:
+                metrics = randomSubsetOf(
+                    Arrays.asList(
+                        AucRocMetricResultTests.randomResult(),
+                        PrecisionMetricResultTests.randomResult(),
+                        RecallMetricResultTests.randomResult(),
+                        ConfusionMatrixMetricResultTests.randomResult()));
+                break;
+            case Regression.NAME:
+                metrics = randomSubsetOf(
+                    Arrays.asList(
+                        MeanSquaredErrorMetricResultTests.randomResult(),
+                        RSquaredMetricResultTests.randomResult()));
+                break;
+            case Classification.NAME:
+                metrics = randomSubsetOf(
+                    Arrays.asList(
+                        AccuracyMetricResultTests.randomResult(),
+                        MulticlassConfusionMatrixMetricResultTests.randomResult()));
+                break;
+            default:
+                throw new AssertionError("Please add missing \"case\" variant to the \"switch\" statement");
         }
-        if (randomBoolean()) {
-            metrics.add(PrecisionMetricResultTests.randomResult());
-        }
-        if (randomBoolean()) {
-            metrics.add(RecallMetricResultTests.randomResult());
-        }
-        if (randomBoolean()) {
-            metrics.add(ConfusionMatrixMetricResultTests.randomResult());
-        }
-        if (randomBoolean()) {
-            metrics.add(MeanSquaredErrorMetricResultTests.randomResult());
-        }
-        return new EvaluateDataFrameResponse(randomAlphaOfLength(5), metrics);
+        return new EvaluateDataFrameResponse(evaluationName, metrics);
     }
 
     @Override

diff --git a/...lasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java b/...lasticsearch/client/ml/dataframe/evaluation/classification/AccuracyMetricResultTests.java
@@ -31,15 +31,14 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-public class AccuracyMetricResultTests extends AbstractXContentTestCase<AccuracyMetric.Result> {
+public class AccuracyMetricResultTests extends AbstractXContentTestCase<Result> {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
     }
 
-    @Override
-    protected AccuracyMetric.Result createTestInstance() {
+    public static Result randomResult() {
         int numClasses = randomIntBetween(2, 100);
         List<String> classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList());
         List<ActualClass> actualClasses = new ArrayList<>(numClasses);
@@ -52,8 +51,13 @@ protected AccuracyMetric.Result createTestInstance() {
     }
 
     @Override
-    protected AccuracyMetric.Result doParseInstance(XContentParser parser) throws IOException {
-        return AccuracyMetric.Result.fromXContent(parser);
+    protected Result createTestInstance() {
+        return randomResult();
+    }
+
+    @Override
+    protected Result doParseInstance(XContentParser parser) throws IOException {
+        return Result.fromXContent(parser);
     }
 
     @Override

diff --git a/.../org/elasticsearch/client/ml/dataframe/evaluation/classification/ClassificationTests.java b/.../org/elasticsearch/client/ml/dataframe/evaluation/classification/ClassificationTests.java
@@ -38,7 +38,10 @@ protected NamedXContentRegistry xContentRegistry() {
 
     static Classification createRandom() {
         List<EvaluationMetric> metrics =
-            randomSubsetOf(Arrays.asList(AccuracyMetricTests.createRandom(), MulticlassConfusionMatrixMetricTests.createRandom()));
+            randomSubsetOf(
+                Arrays.asList(
+                    AccuracyMetricTests.createRandom(),
+                    MulticlassConfusionMatrixMetricTests.createRandom()));
         return new Classification(randomAlphaOfLength(10), randomAlphaOfLength(10), metrics.isEmpty() ? null : metrics);
     }
 

diff --git a/...nt/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixMetricResultTests.java b/...nt/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixMetricResultTests.java
@@ -40,8 +40,7 @@ protected NamedXContentRegistry xContentRegistry() {
         return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
     }
 
-    @Override
-    protected Result createTestInstance() {
+    public static Result randomResult() {
         int numClasses = randomIntBetween(2, 100);
         List<String> classNames = Stream.generate(() -> randomAlphaOfLength(10)).limit(numClasses).collect(Collectors.toList());
         List<ActualClass> actualClasses = new ArrayList<>(numClasses);
@@ -60,6 +59,11 @@ protected Result createTestInstance() {
         return new Result(actualClasses, randomBoolean() ? randomNonNegativeLong() : null);
     }
 
+    @Override
+    protected Result createTestInstance() {
+        return randomResult();
+    }
+
     @Override
     protected Result doParseInstance(XContentParser parser) throws IOException {
         return Result.fromXContent(parser);

diff --git a/...ient/ml/AucRocMetricAucRocPointTests.java → ...ication/AucRocMetricAucRocPointTests.java b/...ient/ml/AucRocMetricAucRocPointTests.java → ...ication/AucRocMetricAucRocPointTests.java
@@ -16,9 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml;
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
 
-import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 

diff --git a/...ch/client/ml/AucRocMetricResultTests.java → ...assification/AucRocMetricResultTests.java b/...ch/client/ml/AucRocMetricResultTests.java → ...assification/AucRocMetricResultTests.java
@@ -16,9 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml;
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
 
-import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
@@ -27,11 +26,11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.client.ml.AucRocMetricAucRocPointTests.randomPoint;
+import static org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetricAucRocPointTests.randomPoint;
 
 public class AucRocMetricResultTests extends AbstractXContentTestCase<AucRocMetric.Result> {
 
-    static AucRocMetric.Result randomResult() {
+    public static AucRocMetric.Result randomResult() {
         return new AucRocMetric.Result(
             randomDouble(),
             Stream

diff --git a/...sionMatrixMetricConfusionMatrixTests.java → ...sionMatrixMetricConfusionMatrixTests.java b/...sionMatrixMetricConfusionMatrixTests.java → ...sionMatrixMetricConfusionMatrixTests.java
@@ -16,9 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml;
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
 
-import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 

diff --git a/.../ml/ConfusionMatrixMetricResultTests.java → ...ion/ConfusionMatrixMetricResultTests.java b/.../ml/ConfusionMatrixMetricResultTests.java → ...ion/ConfusionMatrixMetricResultTests.java
@@ -16,9 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.client.ml;
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
 
-import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
@@ -27,11 +26,11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.client.ml.ConfusionMatrixMetricConfusionMatrixTests.randomConfusionMatrix;
+import static org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetricConfusionMatrixTests.randomConfusionMatrix;
 
 public class ConfusionMatrixMetricResultTests extends AbstractXContentTestCase<ConfusionMatrixMetric.Result> {
 
-    static ConfusionMatrixMetric.Result randomResult() {
+    public static ConfusionMatrixMetric.Result randomResult() {
         return new ConfusionMatrixMetric.Result(
             Stream
                 .generate(() -> randomConfusionMatrix())