[7.x] Implement precision and recall metrics for classification e…

…valuation (#49671) (#50378)
elastic · Dec 19, 2019 · cc4bc79 · cc4bc79
1 parent 9033052
commit cc4bc79
Show file tree

Hide file tree

Showing 54 changed files with 2,487 additions and 382 deletions.
diff --git a/.../rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java b/.../rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java
@@ -35,6 +35,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider.registeredMetricName;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 public class EvaluateDataFrameResponse implements ToXContentObject {
@@ -47,7 +48,7 @@ public static EvaluateDataFrameResponse fromXContent(XContentParser parser) thro
         ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
         String evaluationName = parser.currentName();
         parser.nextToken();
-        Map<String, EvaluationMetric.Result> metrics = parser.map(LinkedHashMap::new, EvaluateDataFrameResponse::parseMetric);
+        Map<String, EvaluationMetric.Result> metrics = parser.map(LinkedHashMap::new, p -> parseMetric(evaluationName, p));
         List<EvaluationMetric.Result> knownMetrics =
             metrics.values().stream()
                 .filter(Objects::nonNull)  // Filter out null values returned by {@link EvaluateDataFrameResponse::parseMetric}.
@@ -56,10 +57,10 @@ public static EvaluateDataFrameResponse fromXContent(XContentParser parser) thro
         return new EvaluateDataFrameResponse(evaluationName, knownMetrics);
     }
 
-    private static EvaluationMetric.Result parseMetric(XContentParser parser) throws IOException {
+    private static EvaluationMetric.Result parseMetric(String evaluationName, XContentParser parser) throws IOException {
         String metricName = parser.currentName();
         try {
-            return parser.namedObject(EvaluationMetric.Result.class, metricName, null);
+            return parser.namedObject(EvaluationMetric.Result.class, registeredMetricName(evaluationName, metricName), null);
         } catch (NamedObjectNotFoundException e) {
             parser.skipChildren();
             // Metric name not recognized. Return {@code null} value here and filter it out later.

diff --git a/...a/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java b/...a/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
@@ -20,24 +20,36 @@
 
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.AccuracyMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.Classification;
-import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.classification.MulticlassConfusionMatrixMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.regression.MeanSquaredErrorMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.RSquaredMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.regression.Regression;
-import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
 import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
 
 import java.util.Arrays;
 import java.util.List;
 
 public class MlEvaluationNamedXContentProvider implements NamedXContentProvider {
 
+    /**
+     * Constructs the name under which a metric (or metric result) is registered.
+     * The name is prefixed with evaluation name so that registered names are unique.
+     *
+     * @param evaluationName name of the evaluation
+     * @param metricName name of the metric
+     * @return name appropriate for registering a metric (or metric result) in {@link NamedXContentRegistry}
+     */
+    public static String registeredMetricName(String evaluationName, String metricName) {
+        return evaluationName + "." +  metricName;
+    }
+
     @Override
     public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
         return Arrays.asList(
@@ -47,39 +59,91 @@ Evaluation.class, new ParseField(BinarySoftClassification.NAME), BinarySoftClass
             new NamedXContentRegistry.Entry(Evaluation.class, new ParseField(Classification.NAME), Classification::fromXContent),
             new NamedXContentRegistry.Entry(Evaluation.class, new ParseField(Regression.NAME), Regression::fromXContent),
             // Evaluation metrics
-            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(AucRocMetric.NAME), AucRocMetric::fromXContent),
-            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionMetric.NAME), PrecisionMetric::fromXContent),
-            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(RecallMetric.NAME), RecallMetric::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.class, new ParseField(ConfusionMatrixMetric.NAME), ConfusionMatrixMetric::fromXContent),
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, AucRocMetric.NAME)),
+                AucRocMetric::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.class, new ParseField(AccuracyMetric.NAME), AccuracyMetric::fromXContent),
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, PrecisionMetric.NAME)),
+                PrecisionMetric::fromXContent),
             new NamedXContentRegistry.Entry(
                 EvaluationMetric.class,
-                new ParseField(MulticlassConfusionMatrixMetric.NAME),
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, RecallMetric.NAME)),
+                RecallMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, ConfusionMatrixMetric.NAME)),
+                ConfusionMatrixMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(Classification.NAME, AccuracyMetric.NAME)),
+                AccuracyMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(
+                    Classification.NAME, org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric.NAME)),
+                org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(
+                    Classification.NAME, org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric.NAME)),
+                org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(Classification.NAME, MulticlassConfusionMatrixMetric.NAME)),
                 MulticlassConfusionMatrixMetric::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.class, new ParseField(MeanSquaredErrorMetric.NAME), MeanSquaredErrorMetric::fromXContent),
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(Regression.NAME, MeanSquaredErrorMetric.NAME)),
+                MeanSquaredErrorMetric::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.class, new ParseField(RSquaredMetric.NAME), RSquaredMetric::fromXContent),
+                EvaluationMetric.class,
+                new ParseField(registeredMetricName(Regression.NAME, RSquaredMetric.NAME)),
+                RSquaredMetric::fromXContent),
             // Evaluation metrics results
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(AucRocMetric.NAME), AucRocMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, AucRocMetric.NAME)),
+                AucRocMetric.Result::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, PrecisionMetric.NAME)),
+                PrecisionMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(PrecisionMetric.NAME), PrecisionMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, RecallMetric.NAME)),
+                RecallMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(RecallMetric.NAME), RecallMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(BinarySoftClassification.NAME, ConfusionMatrixMetric.NAME)),
+                ConfusionMatrixMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(ConfusionMatrixMetric.NAME), ConfusionMatrixMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(Classification.NAME, AccuracyMetric.NAME)),
+                AccuracyMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(AccuracyMetric.NAME), AccuracyMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(
+                    Classification.NAME, org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric.NAME)),
+                org.elasticsearch.client.ml.dataframe.evaluation.classification.PrecisionMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
                 EvaluationMetric.Result.class,
-                new ParseField(MulticlassConfusionMatrixMetric.NAME),
+                new ParseField(registeredMetricName(
+                    Classification.NAME, org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric.NAME)),
+                org.elasticsearch.client.ml.dataframe.evaluation.classification.RecallMetric.Result::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(Classification.NAME, MulticlassConfusionMatrixMetric.NAME)),
                 MulticlassConfusionMatrixMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(MeanSquaredErrorMetric.NAME), MeanSquaredErrorMetric.Result::fromXContent),
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(Regression.NAME, MeanSquaredErrorMetric.NAME)),
+                MeanSquaredErrorMetric.Result::fromXContent),
             new NamedXContentRegistry.Entry(
-                EvaluationMetric.Result.class, new ParseField(RSquaredMetric.NAME), RSquaredMetric.Result::fromXContent));
+                EvaluationMetric.Result.class,
+                new ParseField(registeredMetricName(Regression.NAME, RSquaredMetric.NAME)),
+                RSquaredMetric.Result::fromXContent)
+        );
     }
 }
diff --git a/.../java/org/elasticsearch/client/ml/dataframe/evaluation/classification/Classification.java b/.../java/org/elasticsearch/client/ml/dataframe/evaluation/classification/Classification.java
@@ -32,6 +32,10 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider.registeredMetricName;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 /**
  * Evaluation of classification results.
  */
@@ -48,10 +52,10 @@ public class Classification implements Evaluation {
         NAME, true, a -> new Classification((String) a[0], (String) a[1], (List<EvaluationMetric>) a[2]));
 
     static {
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), ACTUAL_FIELD);
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), PREDICTED_FIELD);
-        PARSER.declareNamedObjects(ConstructingObjectParser.optionalConstructorArg(),
-            (p, c, n) -> p.namedObject(EvaluationMetric.class, n, c), METRICS);
+        PARSER.declareString(constructorArg(), ACTUAL_FIELD);
+        PARSER.declareString(constructorArg(), PREDICTED_FIELD);
+        PARSER.declareNamedObjects(
+            optionalConstructorArg(), (p, c, n) -> p.namedObject(EvaluationMetric.class, registeredMetricName(NAME, n), c), METRICS);
     }
 
     public static Classification fromXContent(XContentParser parser) {