Benchmarking tool with Comet (#545)

* comet benchmarking enabled

* updated BM docs

* tweaked comment

* commen changet

* fixed end of file

Co-authored-by: Samet Akcay <samet.akcay@intel.com>

docs/source/guides/benchmarking.rst

@@ -3,7 +3,7 @@
 Benchmarking
 =============
 
-To add to the suit of experiment tracking and optimization, anomalib also includes a benchmarking script for gathering results across different combinations of models, their parameters, and dataset categories. The model performance and throughputs are logged into a csv file that can also serve as a means to track model drift. Optionally, these same results can be logged to Weights and Biases and TensorBoard. A sample configuration file is shown below.
+To add to the suit of experiment tracking and optimization, anomalib also includes a benchmarking script for gathering results across different combinations of models, their parameters, and dataset categories. The model performance and throughputs are logged into a csv file that can also serve as a means to track model drift. Optionally, these same results can be logged to Comet, Weights and Biases and TensorBoard. A sample configuration file is shown below.
 
 .. code-block:: yaml
 
@@ -13,6 +13,7 @@ To add to the suit of experiment tracking and optimization, anomalib also includ
     - cpu
     - gpu
   writer:
+    - comet
     - wandb
     - tensorboard
   grid_search:

tools/benchmarking/benchmark.py

@@ -21,7 +21,7 @@
 import torch
 from omegaconf import DictConfig, ListConfig, OmegaConf
 from pytorch_lightning import Trainer, seed_everything
-from utils import convert_to_openvino, upload_to_wandb, write_metrics
+from utils import convert_to_openvino, upload_to_comet, upload_to_wandb, write_metrics
 
 from anomalib.config import get_configurable_parameters, update_input_size_config
 from anomalib.data import get_datamodule
@@ -225,6 +225,8 @@ def distribute(config: Union[DictConfig, ListConfig]):
         distribute_over_gpus(config, folder=runs_folder)
     if "wandb" in config.writer:
         upload_to_wandb(team="anomalib", folder=runs_folder)
+    if "comet" in config.writer:
+        upload_to_comet(folder=runs_folder)
 
 
 def sweep(

tools/benchmarking/utils/__init__.py

@@ -4,6 +4,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from .convert import convert_to_openvino
-from .metrics import upload_to_wandb, write_metrics
+from .metrics import upload_to_comet, upload_to_wandb, write_metrics
 
-__all__ = ["convert_to_openvino", "write_metrics", "upload_to_wandb"]
+__all__ = ["convert_to_openvino", "write_metrics", "upload_to_comet", "upload_to_wandb"]

tools/benchmarking/utils/metrics.py

@@ -10,6 +10,7 @@
 from typing import Dict, List, Optional, Union
 
 import pandas as pd
+from comet_ml import Experiment
 from torch.utils.tensorboard.writer import SummaryWriter
 
 import wandb
@@ -116,3 +117,29 @@ def upload_to_wandb(
             )
             wandb.log(row)
             wandb.finish()
+
+
+def upload_to_comet(
+    folder: Optional[str] = None,
+):
+    """Upload the data in csv files to comet.
+
+    Creates a project named benchmarking_[two random characters]. This is so that the project names are unique.
+    One issue is that it does not check for collision
+
+    Args:
+        folder (optional, str): Sub-directory from which runs are picked up. Defaults to None. If none picks from runs.
+    """
+    project = f"benchmarking_{get_unique_key(2)}"
+    tag_list = ["dataset.category", "model_name", "dataset.image_size", "model.backbone", "device"]
+    search_path = "runs/*.csv" if folder is None else f"runs/{folder}/*.csv"
+    for csv_file in glob(search_path):
+        table = pd.read_csv(csv_file)
+        for index, row in table.iterrows():
+            row = dict(row[1:])  # remove index column
+            tags = [str(row[column]) for column in tag_list if column in row.keys()]
+            experiment = Experiment(project_name=project)
+            experiment.set_name(f"{row['model_name']}_{row['dataset.category']}_{index}")
+            experiment.log_metrics(row, step=1, epoch=1)  # populates auto-generated charts on panel view
+            experiment.add_tags(tags)
+            experiment.log_table(filename=csv_file)