From 5e6ef5973500be980e0a1043957668a66c16eedf Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Thu, 12 Jan 2023 11:27:52 +0100 Subject: [PATCH 1/5] Added ClearML instance segmentation and classification support --- utils/loggers/__init__.py | 45 ++++++++++++++++++++---- utils/loggers/clearml/clearml_utils.py | 48 +++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 22da87034f24..cdcbacb57798 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -243,9 +243,7 @@ def on_fit_epoch_end(self, vals, epoch, best_fitness, fi): for k, v in x.items(): self.tb.add_scalar(k, v, epoch) elif self.clearml: # log to ClearML if TensorBoard not used - for k, v in x.items(): - title, series = k.split('/') - self.clearml.task.get_logger().report_scalar(title, series, v, epoch) + self.clearml.log_scalars(x, epoch) if self.wandb: if best_fitness == fi: @@ -299,9 +297,13 @@ def on_train_end(self, last, best, epoch, results): self.wandb.finish_run() if self.clearml and not self.opt.evolve: - self.clearml.task.update_output_model(model_path=str(best if best.exists() else last), - name='Best Model', - auto_delete_file=False) + self.clearml.log_summary(dict(zip(self.keys[3:10], results))) + self.clearml.log_debug_samples(files, title="Results") + self.clearml.log_model( + str(best if best.exists() else last), + "Best Model" if best.exists() else "Last Model", + epoch + ) if self.comet_logger: final_results = dict(zip(self.keys[3:10], results)) @@ -313,6 +315,8 @@ def on_params_update(self, params: dict): self.wandb.wandb_run.config.update(params, allow_val_change=True) if self.comet_logger: self.comet_logger.on_params_update(params) + if self.clearml: + self.clearml.task.connect(params) class GenericLogger: @@ -325,7 +329,7 @@ class GenericLogger: include: loggers to include """ - def __init__(self, opt, console_logger, include=('tb', 'wandb')): + def __init__(self, opt, console_logger, include=('tb', 'wandb', 'clearml')): # init default loggers self.save_dir = Path(opt.save_dir) self.include = include @@ -343,6 +347,22 @@ def __init__(self, opt, console_logger, include=('tb', 'wandb')): config=opt) else: self.wandb = None + + if clearml and 'clearml' in self.include: + try: + if 'hyp' not in opt: + hyp = {} + else: + hyp = opt.hyp + self.clearml = ClearmlLogger(opt, hyp) + except Exception: + self.clearml = None + prefix = colorstr('ClearML: ') + LOGGER.warning(f'{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging.' + f' See https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml#readme') + else: + self.clearml = None + def log_metrics(self, metrics, epoch): # Log metrics dictionary to all loggers @@ -359,6 +379,10 @@ def log_metrics(self, metrics, epoch): if self.wandb: self.wandb.log(metrics, step=epoch) + + if self.clearml: + self.clearml.log_scalars(metrics, epoch) + def log_images(self, files, name='Images', epoch=0): # Log images to all loggers @@ -371,6 +395,9 @@ def log_images(self, files, name='Images', epoch=0): if self.wandb: self.wandb.log({name: [wandb.Image(str(f), caption=f.name) for f in files]}, step=epoch) + + if self.clearml: + self.clearml.log_debug_samples(files, title=name) def log_graph(self, model, imgsz=(640, 640)): # Log model graph to all loggers @@ -383,11 +410,15 @@ def log_model(self, model_path, epoch=0, metadata={}): art = wandb.Artifact(name=f"run_{wandb.run.id}_model", type="model", metadata=metadata) art.add_file(str(model_path)) wandb.log_artifact(art) + if self.clearml: + self.clearml.log_model(model_path=model_path, model_name=model_path.stem) def update_params(self, params): # Update the paramters logged if self.wandb: wandb.run.config.update(params, allow_val_change=True) + if self.clearml: + self.clearml.task.connect(params) def log_tensorboard_graph(tb, model, imgsz=(640, 640)): diff --git a/utils/loggers/clearml/clearml_utils.py b/utils/loggers/clearml/clearml_utils.py index 3457727a96a4..74431a44b973 100644 --- a/utils/loggers/clearml/clearml_utils.py +++ b/utils/loggers/clearml/clearml_utils.py @@ -79,14 +79,16 @@ def __init__(self, opt, hyp): # Maximum number of images to log to clearML per epoch self.max_imgs_to_log_per_epoch = 16 # Get the interval of epochs when bounding box images should be logged - self.bbox_interval = opt.bbox_interval + # Only for detection task though! + if 'bbox_interval' in opt: + self.bbox_interval = opt.bbox_interval self.clearml = clearml self.task = None self.data_dict = None if self.clearml: self.task = Task.init( - project_name=opt.project if opt.project != 'runs/train' else 'YOLOv5', - task_name=opt.name if opt.name != 'exp' else 'Training', + project_name=opt.project if not str(opt.project).startswith('runs/') else 'YOLOv5', + task_name=opt.name if opt.name != 'exp' else "Training", tags=['YOLOv5'], output_uri=True, reuse_last_task_id=opt.exist_ok, @@ -112,6 +114,44 @@ def __init__(self, opt, hyp): # Set data to data_dict because wandb will crash without this information and opt is the best way # to give it to them opt.data = self.data_dict + + def log_scalars(self, metrics, epoch): + """ + Log scalars/metrics to ClearML + + arguments: + metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...} + epoch (int) iteration number for the current set of metrics + """ + for k, v in metrics.items(): + title, series = k.split('/') + self.task.get_logger().report_scalar(title, series, v, epoch) + + def log_model(self, model_path, model_name, epoch=0): + """ + Log model weights to ClearML + + arguments: + model_path (PosixPath or str) Path to the model weights + model_name (str) Name of the model visible in ClearML + epoch (int) Iteration / epoch of the model weights + """ + self.task.update_output_model( + model_path=str(model_path), + name=model_name, + iteration=epoch, + auto_delete_file=False + ) + + def log_summary(self, metrics): + """ + Log final metrics to a summary table + + arguments: + metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...} + """ + for k, v in metrics.items(): + self.task.get_logger().report_single_value(k, v) def log_debug_samples(self, files, title='Debug Samples'): """ @@ -126,7 +166,7 @@ def log_debug_samples(self, files, title='Debug Samples'): it = re.search(r'_batch(\d+)', f.name) iteration = int(it.groups()[0]) if it else 0 self.task.get_logger().report_image(title=title, - series=f.name.replace(it.group(), ''), + series=f.name.replace(f"_batch{iteration}", ''), local_path=str(f), iteration=iteration) From 5af4f2c28d483130b2a3e92b5a864ccf6d43bb01 Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Thu, 12 Jan 2023 14:16:15 +0100 Subject: [PATCH 2/5] Cleaned up ClearML plot output --- utils/loggers/__init__.py | 7 ++++--- utils/loggers/clearml/clearml_utils.py | 20 +++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index cdcbacb57798..0c82e3a4e5fd 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -169,10 +169,11 @@ def on_pretrain_routine_end(self, labels, names): paths = self.save_dir.glob('*labels*.jpg') # training labels if self.wandb: self.wandb.log({"Labels": [wandb.Image(str(x), caption=x.name) for x in paths]}) - # if self.clearml: - # pass # ClearML saves these images automatically using hooks if self.comet_logger: self.comet_logger.on_pretrain_routine_end(paths) + if self.clearml: + for path in paths: + self.clearml.log_plot(title=path.stem, plot_path=path) def on_train_batch_end(self, model, ni, imgs, targets, paths, vals): log_dict = dict(zip(self.keys[0:3], vals)) @@ -298,7 +299,7 @@ def on_train_end(self, last, best, epoch, results): if self.clearml and not self.opt.evolve: self.clearml.log_summary(dict(zip(self.keys[3:10], results))) - self.clearml.log_debug_samples(files, title="Results") + [self.clearml.log_plot(title=f.stem, plot_path=f) for f in files] self.clearml.log_model( str(best if best.exists() else last), "Best Model" if best.exists() else "Last Model", diff --git a/utils/loggers/clearml/clearml_utils.py b/utils/loggers/clearml/clearml_utils.py index 74431a44b973..fbbf54102107 100644 --- a/utils/loggers/clearml/clearml_utils.py +++ b/utils/loggers/clearml/clearml_utils.py @@ -3,6 +3,8 @@ import re from pathlib import Path +import matplotlib.pyplot as plt +import matplotlib.image as mpimg import numpy as np import yaml @@ -92,7 +94,7 @@ def __init__(self, opt, hyp): tags=['YOLOv5'], output_uri=True, reuse_last_task_id=opt.exist_ok, - auto_connect_frameworks={'pytorch': False} + auto_connect_frameworks={'pytorch': False, 'matplotlib': False} # We disconnect pytorch auto-detection, because we added manual model save points in the code ) # ClearML's hooks will already grab all general parameters @@ -152,6 +154,22 @@ def log_summary(self, metrics): """ for k, v in metrics.items(): self.task.get_logger().report_single_value(k, v) + + def log_plot(self, title, plot_path): + """ + Log image as plot in the plot section of ClearML + + arguments: + title (str) Title of the plot + plot_path (PosixPath or str) Path to the saved image file + """ + img = mpimg.imread(plot_path) + fig = plt.figure() + ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', + xticks=[], yticks=[]) # no ticks + ax.imshow(img) + + self.task.get_logger().report_matplotlib_figure(title, "", figure=fig, report_interactive=False) def log_debug_samples(self, files, title='Debug Samples'): """ From 6e64e8aef424f47dad3b292053f6d956e5d62e7d Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Thu, 12 Jan 2023 14:19:26 +0100 Subject: [PATCH 3/5] typos --- utils/loggers/__init__.py | 1 + utils/loggers/clearml/clearml_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 0c82e3a4e5fd..a53cdb653568 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -351,6 +351,7 @@ def __init__(self, opt, console_logger, include=('tb', 'wandb', 'clearml')): if clearml and 'clearml' in self.include: try: + # Hyp is not available in classification mode if 'hyp' not in opt: hyp = {} else: diff --git a/utils/loggers/clearml/clearml_utils.py b/utils/loggers/clearml/clearml_utils.py index fbbf54102107..ab69d38acdb0 100644 --- a/utils/loggers/clearml/clearml_utils.py +++ b/utils/loggers/clearml/clearml_utils.py @@ -90,7 +90,7 @@ def __init__(self, opt, hyp): if self.clearml: self.task = Task.init( project_name=opt.project if not str(opt.project).startswith('runs/') else 'YOLOv5', - task_name=opt.name if opt.name != 'exp' else "Training", + task_name=opt.name if opt.name != 'exp' else 'Training', tags=['YOLOv5'], output_uri=True, reuse_last_task_id=opt.exist_ok, From b2bece02c8188b722451707cc1f889f78d420200 Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Thu, 12 Jan 2023 14:38:33 +0100 Subject: [PATCH 4/5] Log results as plots instead of debug samples --- utils/loggers/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index a53cdb653568..ae013e081fa9 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -399,7 +399,10 @@ def log_images(self, files, name='Images', epoch=0): self.wandb.log({name: [wandb.Image(str(f), caption=f.name) for f in files]}, step=epoch) if self.clearml: - self.clearml.log_debug_samples(files, title=name) + if name == 'Results': + [self.clearml.log_plot(f.stem, f) for f in files] + else: + self.clearml.log_debug_samples(files, title=name) def log_graph(self, model, imgsz=(640, 640)): # Log model graph to all loggers From aea30092e06b50dcb9fb1be56624aa71c53c8bc6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:39:44 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- utils/loggers/__init__.py | 17 +++++------- utils/loggers/clearml/clearml_utils.py | 37 +++++++++++++------------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index ae013e081fa9..70b512c402a5 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -300,11 +300,8 @@ def on_train_end(self, last, best, epoch, results): if self.clearml and not self.opt.evolve: self.clearml.log_summary(dict(zip(self.keys[3:10], results))) [self.clearml.log_plot(title=f.stem, plot_path=f) for f in files] - self.clearml.log_model( - str(best if best.exists() else last), - "Best Model" if best.exists() else "Last Model", - epoch - ) + self.clearml.log_model(str(best if best.exists() else last), + "Best Model" if best.exists() else "Last Model", epoch) if self.comet_logger: final_results = dict(zip(self.keys[3:10], results)) @@ -348,14 +345,14 @@ def __init__(self, opt, console_logger, include=('tb', 'wandb', 'clearml')): config=opt) else: self.wandb = None - + if clearml and 'clearml' in self.include: try: # Hyp is not available in classification mode if 'hyp' not in opt: hyp = {} else: - hyp = opt.hyp + hyp = opt.hyp self.clearml = ClearmlLogger(opt, hyp) except Exception: self.clearml = None @@ -364,7 +361,6 @@ def __init__(self, opt, console_logger, include=('tb', 'wandb', 'clearml')): f' See https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml#readme') else: self.clearml = None - def log_metrics(self, metrics, epoch): # Log metrics dictionary to all loggers @@ -381,11 +377,10 @@ def log_metrics(self, metrics, epoch): if self.wandb: self.wandb.log(metrics, step=epoch) - + if self.clearml: self.clearml.log_scalars(metrics, epoch) - def log_images(self, files, name='Images', epoch=0): # Log images to all loggers files = [Path(f) for f in (files if isinstance(files, (tuple, list)) else [files])] # to Path @@ -397,7 +392,7 @@ def log_images(self, files, name='Images', epoch=0): if self.wandb: self.wandb.log({name: [wandb.Image(str(f), caption=f.name) for f in files]}, step=epoch) - + if self.clearml: if name == 'Results': [self.clearml.log_plot(f.stem, f) for f in files] diff --git a/utils/loggers/clearml/clearml_utils.py b/utils/loggers/clearml/clearml_utils.py index ab69d38acdb0..d0d52cfd0968 100644 --- a/utils/loggers/clearml/clearml_utils.py +++ b/utils/loggers/clearml/clearml_utils.py @@ -3,8 +3,8 @@ import re from pathlib import Path -import matplotlib.pyplot as plt import matplotlib.image as mpimg +import matplotlib.pyplot as plt import numpy as np import yaml @@ -94,7 +94,9 @@ def __init__(self, opt, hyp): tags=['YOLOv5'], output_uri=True, reuse_last_task_id=opt.exist_ok, - auto_connect_frameworks={'pytorch': False, 'matplotlib': False} + auto_connect_frameworks={ + 'pytorch': False, + 'matplotlib': False} # We disconnect pytorch auto-detection, because we added manual model save points in the code ) # ClearML's hooks will already grab all general parameters @@ -116,11 +118,11 @@ def __init__(self, opt, hyp): # Set data to data_dict because wandb will crash without this information and opt is the best way # to give it to them opt.data = self.data_dict - + def log_scalars(self, metrics, epoch): """ Log scalars/metrics to ClearML - + arguments: metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...} epoch (int) iteration number for the current set of metrics @@ -128,47 +130,44 @@ def log_scalars(self, metrics, epoch): for k, v in metrics.items(): title, series = k.split('/') self.task.get_logger().report_scalar(title, series, v, epoch) - + def log_model(self, model_path, model_name, epoch=0): """ Log model weights to ClearML - + arguments: model_path (PosixPath or str) Path to the model weights model_name (str) Name of the model visible in ClearML epoch (int) Iteration / epoch of the model weights """ - self.task.update_output_model( - model_path=str(model_path), - name=model_name, - iteration=epoch, - auto_delete_file=False - ) - + self.task.update_output_model(model_path=str(model_path), + name=model_name, + iteration=epoch, + auto_delete_file=False) + def log_summary(self, metrics): """ Log final metrics to a summary table - + arguments: metrics (dict) Metrics in dict format: {"metrics/mAP": 0.8, ...} """ for k, v in metrics.items(): self.task.get_logger().report_single_value(k, v) - + def log_plot(self, title, plot_path): """ Log image as plot in the plot section of ClearML - + arguments: title (str) Title of the plot plot_path (PosixPath or str) Path to the saved image file """ img = mpimg.imread(plot_path) fig = plt.figure() - ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', - xticks=[], yticks=[]) # no ticks + ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', xticks=[], yticks=[]) # no ticks ax.imshow(img) - + self.task.get_logger().report_matplotlib_figure(title, "", figure=fig, report_interactive=False) def log_debug_samples(self, files, title='Debug Samples'):