🐞 Log benchmarking results in sub folder (#483)

Log benchmarking results in sub folder

tools/benchmarking/benchmark.py

@@ -25,9 +25,10 @@
 import warnings
 from argparse import ArgumentParser
 from concurrent.futures import ProcessPoolExecutor, as_completed
+from datetime import datetime
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Dict, List, Union, cast
+from typing import Dict, List, Optional, Union, cast
 
 import torch
 from omegaconf import DictConfig, ListConfig, OmegaConf
@@ -147,18 +148,19 @@ def get_single_model_metrics(model_config: Union[DictConfig, ListConfig], openvi
     return data
 
 
-def compute_on_cpu(sweep_config: Union[DictConfig, ListConfig]):
+def compute_on_cpu(sweep_config: Union[DictConfig, ListConfig], folder: Optional[str] = None):
     """Compute all run configurations over a sigle CPU."""
     for run_config in get_run_config(sweep_config.grid_search):
         model_metrics = sweep(run_config, 0, sweep_config.seed, False)
-        write_metrics(model_metrics, sweep_config.writer)
+        write_metrics(model_metrics, sweep_config.writer, folder)
 
 
 def compute_on_gpu(
     run_configs: List[DictConfig],
     device: int,
     seed: int,
     writers: List[str],
+    folder: Optional[str] = None,
     compute_openvino: bool = False,
 ):
     """Go over each run config and collect the result.
@@ -168,19 +170,20 @@ def compute_on_gpu(
         device (int): The GPU id used for running the sweep.
         seed (int): Fix a seed.
         writers (List[str]): Destinations to write to.
+        folder (optional, str): Sub-directory to which runs are written to. Defaults to None. If none writes to root.
         compute_openvino (bool, optional): Compute OpenVINO throughput. Defaults to False.
     """
     for run_config in run_configs:
         if isinstance(run_config, (DictConfig, ListConfig)):
             model_metrics = sweep(run_config, device, seed, compute_openvino)
-            write_metrics(model_metrics, writers)
+            write_metrics(model_metrics, writers, folder)
         else:
             raise ValueError(
                 f"Expecting `run_config` of type DictConfig or ListConfig. Got {type(run_config)} instead."
             )
 
 
-def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig]):
+def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig], folder: Optional[str] = None):
     """Distribute metric collection over all available GPUs. This is done by splitting the list of configurations."""
     with ProcessPoolExecutor(
         max_workers=torch.cuda.device_count(), mp_context=multiprocessing.get_context("spawn")
@@ -197,6 +200,7 @@ def distribute_over_gpus(sweep_config: Union[DictConfig, ListConfig]):
                     device_id + 1,
                     sweep_config.seed,
                     sweep_config.writer,
+                    folder,
                     sweep_config.compute_openvino,
                 )
             )
@@ -214,24 +218,28 @@ def distribute(config: Union[DictConfig, ListConfig]):
         config: (Union[DictConfig, ListConfig]): Sweep configuration.
     """
 
+    runs_folder = datetime.strftime(datetime.now(), "%Y_%m_%d-%H_%M_%S")
     devices = config.hardware
     if not torch.cuda.is_available() and "gpu" in devices:
         pl_logger.warning("Config requested GPU benchmarking but torch could not detect any cuda enabled devices")
     elif {"cpu", "gpu"}.issubset(devices):
         # Create process for gpu and cpu
         with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn")) as executor:
-            jobs = [executor.submit(compute_on_cpu, config), executor.submit(distribute_over_gpus, config)]
+            jobs = [
+                executor.submit(compute_on_cpu, config, runs_folder),
+                executor.submit(distribute_over_gpus, config, runs_folder),
+            ]
             for job in as_completed(jobs):
                 try:
                     job.result()
                 except Exception as exception:
                     raise Exception(f"Error occurred while computing benchmark on device {job}") from exception
     elif "cpu" in devices:
-        compute_on_cpu(config)
+        compute_on_cpu(config, folder=runs_folder)
     elif "gpu" in devices:
-        distribute_over_gpus(config)
+        distribute_over_gpus(config, folder=runs_folder)
     if "wandb" in config.writer:
-        upload_to_wandb(team="anomalib")
+        upload_to_wandb(team="anomalib", folder=runs_folder)
 
 
 def sweep(

tools/benchmarking/utils/metrics.py

@@ -17,28 +17,34 @@
 import string
 from glob import glob
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 import pandas as pd
 from torch.utils.tensorboard.writer import SummaryWriter
 
 import wandb
 
 
-def write_metrics(model_metrics: Dict[str, Union[str, float]], writers: List[str]):
+def write_metrics(
+    model_metrics: Dict[str, Union[str, float]],
+    writers: List[str],
+    folder: Optional[str] = None,
+):
     """Writes metrics to destination provided in the sweep config.
 
     Args:
         model_metrics (Dict): Dictionary to be written
         writers (List[str]): List of destinations.
+        folder (optional, str): Sub-directory to which runs are written to. Defaults to None. If none writes to root.
     """
     # Write to file as each run is computed
     if model_metrics == {} or model_metrics is None:
         return
 
     # Write to CSV
     metrics_df = pd.DataFrame(model_metrics, index=[0])
-    result_path = Path(f"runs/{model_metrics['model_name']}_{model_metrics['device']}.csv")
+    result_folder = Path("runs") if folder is None else Path(f"runs/{folder}")
+    result_path = result_folder / f"{model_metrics['model_name']}_{model_metrics['device']}.csv"
     Path.mkdir(result_path.parent, parents=True, exist_ok=True)
     if not result_path.is_file():
         metrics_df.to_csv(result_path)
@@ -93,7 +99,10 @@ def get_unique_key(str_len: int) -> str:
     return "".join([random.choice(string.ascii_lowercase) for _ in range(str_len)])
 
 
-def upload_to_wandb(team: str = "anomalib"):
+def upload_to_wandb(
+    team: str = "anomalib",
+    folder: Optional[str] = None,
+):
     """Upload the data in csv files to wandb.
 
     Creates a project named benchmarking_[two random characters]. This is so that the project names are unique.
@@ -102,10 +111,12 @@ def upload_to_wandb(team: str = "anomalib"):
     Args:
         team (str, optional): Name of the team on wandb. This can also be the id of your personal account.
         Defaults to "anomalib".
+        folder (optional, str): Sub-directory from which runs are picked up. Defaults to None. If none picks from runs.
     """
     project = f"benchmarking_{get_unique_key(2)}"
     tag_list = ["dataset.category", "model_name", "dataset.image_size", "model.backbone", "device"]
-    for csv_file in glob("runs/*.csv"):
+    search_path = "runs/*.csv" if folder is None else f"runs/{folder}/*.csv"
+    for csv_file in glob(search_path):
         table = pd.read_csv(csv_file)
         for index, row in table.iterrows():
             row = dict(row[1:])  # remove index column