Rename perf metric evaluator (#191)

* rename performance metrics * update docstring * fix style
huggingface · Jul 20, 2022 · ef3db38 · ef3db38
1 parent 4e7f682
commit ef3db38
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 7 deletions.
diff --git a/src/evaluate/evaluator/base.py b/src/evaluate/evaluator/base.py
@@ -111,12 +111,20 @@ def args_metric(*args):
     def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> Dict[str, Any]:
         """
         A utility function computing time performance metrics:
-            - `latency` - pipeline inference runtime for the evaluation data in seconds,
-            - `throughput` - pipeline throughput in the number of samples per second.
+            - `total_time_in_seconds` - pipeline inference runtime for the evaluation data in seconds,
+            - `samples_per_second` - pipeline throughput in the number of samples per second.
+            - `latency_in_seconds` - pipeline inference runtime for the evaluation data in seconds per sample,
+
         """
         latency = end_time - start_time
         throughput = num_samples / latency
-        return {"latency": latency, "throughput": throughput}
+        latency_sample = 1.0 / throughput
+
+        return {
+            "total_time_in_seconds": latency,
+            "samples_per_second": throughput,
+            "latency_in_seconds": latency_sample,
+        }
 
     @abstractmethod
     def predictions_processor(self, *args, **kwargs):

diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
@@ -200,8 +200,9 @@ def test_perf(self):
             random_state=0,
         )
         self.assertEqual(results["accuracy"], 1.0)
-        self.assertAlmostEqual(results["latency"], 0.1, 1)
-        self.assertAlmostEqual(results["throughput"], len(self.data) / results["latency"], 5)
+        self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1)
+        self.assertAlmostEqual(results["samples_per_second"], len(self.data) / results["total_time_in_seconds"], 5)
+        self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(self.data), 5)
 
     def test_bootstrap_and_perf(self):
         data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]})
@@ -222,8 +223,9 @@ def test_bootstrap_and_perf(self):
         self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.333333, 5)
         self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 0.666666, 5)
         self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498285, 5)
-        self.assertAlmostEqual(results["latency"], 0.1, 1)
-        self.assertAlmostEqual(results["throughput"], len(data) / results["latency"], 5)
+        self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1)
+        self.assertAlmostEqual(results["samples_per_second"], len(data) / results["total_time_in_seconds"], 5)
+        self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(data), 5)
 
 
 class TestImageClassificationEvaluator(TestCase):