Support prediction regex match by setting the operator as a postproce…

prepare/metrics/jaccard_index.py

@@ -0,0 +1,28 @@
+from unitxt import add_to_catalog
+from unitxt.metrics import JaccardIndex
+from unitxt.test_utils.metrics import test_metric
+
+metric = JaccardIndex()
+
+predictions = [["A", "B", "C"]]
+references = [[["B", "A", "D"]]]
+
+instance_targets = [
+    {"jaccard_index": 0.5, "score": 0.5, "score_name": "jaccard_index"},
+]
+
+global_target = {
+    "jaccard_index": 0.5,
+    "score": 0.5,
+    "score_name": "jaccard_index",
+}
+
+outputs = test_metric(
+    metric=metric,
+    predictions=predictions,
+    references=references,
+    instance_targets=instance_targets,
+    global_target=global_target,
+)
+
+add_to_catalog(metric, "metrics.jaccard_index", overwrite=True)

prepare/processors/processors.py

@@ -13,6 +13,7 @@
     LowerCase,
     LowerCaseTillPunc,
     MatchClosestOption,
+    RegexParser,
     StanceToProCon,
     StringOrNotString,
     StrToFloatFormat,
@@ -329,3 +330,9 @@
     "processors.extract_mt_bench_judgment",
     overwrite=True,
 )
+
+add_to_catalog(
+    RegexParser(field="prediction", regex=".+", process_every_value=False),
+    "processors.regex_parser_from_prediction",
+    overwrite=True,
+)

prepare/processors/to_list_by_comma.py

@@ -12,3 +12,13 @@
     "processors.to_list_by_comma",
     overwrite=True,
 )
+
+add_to_catalog(
+    SequentialOperator(
+        steps=[
+            ToListByComma(field="references", process_every_value=True),
+        ]
+    ),
+    "processors.to_list_by_comma_from_references",
+    overwrite=True,
+)

src/unitxt/catalog/metrics/jaccard_index.json

@@ -0,0 +1,3 @@
+{
+    "type": "jaccard_index"
+}

src/unitxt/catalog/processors/regex_parser_from_prediction.json

@@ -0,0 +1,6 @@
+{
+    "type": "regex_parser",
+    "field": "prediction",
+    "regex": ".+",
+    "process_every_value": false
+}

src/unitxt/catalog/processors/to_list_by_comma_from_references.json

@@ -0,0 +1,10 @@
+{
+    "type": "sequential_operator",
+    "steps": [
+        {
+            "type": "to_list_by_comma",
+            "field": "references",
+            "process_every_value": true
+        }
+    ]
+}

src/unitxt/metrics.py

@@ -977,6 +977,40 @@
         return result
 
 
+class JaccardIndex(InstanceMetric):
+    reduction_map = {"mean": ["jaccard_index"]}
+    main_score = "jaccard_index"
+    ci_scores = ["jaccard_index"]
+
+    prediction_type = "Any"  # string representation is compared
+
+    def compute(
+        self, references: List[Any], prediction: Any, task_data: List[Dict]
+    ) -> dict:
+        if not isinstance(prediction, set):
+            prediction = set(prediction)
+        references = [set(reference) for reference in references]
+
+        result = {
+            self.main_score: max(
+                [
+                    float(
+                        (len(reference.intersection(prediction)))
+                        / (
+                            len(reference)
+                            + len(prediction)
+                            - len(reference.intersection(prediction))
+                        )
+                    )
+                    for reference in references
+                ]
+            )
+        }
+        result["score"] = result[self.main_score]
+        result["score_name"] = self.main_score
+        return result
+
+
 class MaxAccuracy(Accuracy):
     """Calculate the maximal accuracy over all instances as the global score."""