From 6ec6e659e6e5cbd7973a48c29998516fb1a73178 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 05:17:44 +0000 Subject: [PATCH 01/41] add comet to logger interface --- utils/loggers/__init__.py | 68 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 880039b1914c..61ff34925534 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -17,7 +17,7 @@ from utils.plots import plot_images, plot_labels, plot_results from utils.torch_utils import de_parallel -LOGGERS = ('csv', 'tb', 'wandb', 'clearml') # *.csv, TensorBoard, Weights & Biases, ClearML +LOGGERS = ('csv', 'tb', 'wandb', 'clearml', 'comet_ml') # *.csv, TensorBoard, Weights & Biases, ClearML RANK = int(os.getenv('RANK', -1)) try: @@ -41,6 +41,18 @@ except (ImportError, AssertionError): clearml = None +try: + if RANK not in [0, -1]: + comet_ml = None + else: + import comet_ml + + assert hasattr(comet_ml, '__version__') # verify package import not local dir + from utils.loggers.comet import CometLogger + +except (ModuleNotFoundError, ImportError, AssertionError): + comet_ml = None + class Loggers(): # YOLOv5 Loggers class @@ -80,7 +92,10 @@ def __init__(self, save_dir=None, weights=None, opt=None, hyp=None, logger=None, prefix = colorstr('ClearML: ') s = f"{prefix}run 'pip install clearml' to automatically track, visualize and remotely train YOLOv5 🚀 in ClearML" self.logger.info(s) - + if not comet_ml: + prefix = colorstr('Comet: ') + s = f"{prefix}run 'pip install comet_ml' to automatically track and visualize YOLOv5 🚀 runs in Comet" + self.logger.info(s) # TensorBoard s = self.save_dir if 'tb' in self.include and not self.opt.evolve: @@ -107,9 +122,25 @@ def __init__(self, save_dir=None, weights=None, opt=None, hyp=None, logger=None, else: self.clearml = None + # Comet + if comet_ml and 'comet' in self.include: + if isinstance(self.opt.resume, str) and self.opt.resume.startswith("comet://"): + run_id = self.opt.resume.split("/")[-1] + self.comet_logger = CometLogger(self.opt, self.hyp, run_id=run_id) + + else: + self.comet_logger = CometLogger(self.opt, self.hyp) + + else: + self.comet_logger = None + def on_train_start(self): - # Callback runs on train start - pass + if self.comet_logger: + self.comet_logger.on_train_start() + + def on_pretrain_routine_start(self): + if self.comet_logger: + self.comet_logger.on_pretrain_routine_start() def on_pretrain_routine_end(self, labels, names): # Callback runs on pre-train routine end @@ -120,8 +151,11 @@ def on_pretrain_routine_end(self, labels, names): self.wandb.log({"Labels": [wandb.Image(str(x), caption=x.name) for x in paths]}) # if self.clearml: # pass # ClearML saves these images automatically using hooks + if self.comet_logger: + self.comet_logger.on_pretrain_routine_end(paths) - def on_train_batch_end(self, model, ni, imgs, targets, paths): + def on_train_batch_end(self, model, ni, imgs, targets, paths, vals): + log_dict = dict(zip(self.keys[0:3], vals)) # Callback runs on train batch end # ni: number integrated batches (since train start) if self.plots: @@ -137,11 +171,21 @@ def on_train_batch_end(self, model, ni, imgs, targets, paths): if self.clearml: self.clearml.log_debug_samples(files, title='Mosaics') + if self.comet_logger: + self.comet_logger.on_train_batch_end(log_dict, step=ni) + def on_train_epoch_end(self, epoch): # Callback runs on train epoch end if self.wandb: self.wandb.current_epoch = epoch + 1 + if self.comet_logger: + self.comet_logger.on_train_epoch_end(epoch) + + def on_val_start(self): + if self.comet_logger: + self.comet_logger.on_val_start() + def on_val_image_end(self, pred, predn, path, names, im): # Callback runs on val image end if self.wandb: @@ -149,6 +193,10 @@ def on_val_image_end(self, pred, predn, path, names, im): if self.clearml: self.clearml.log_image_with_boxes(path, pred, names, im) + def on_val_batch_end(self, batch_i, im, targets, paths, shapes, out): + if self.comet_logger: + self.comet_logger.on_val_batch_end(batch_i, im, targets, paths, shapes, out) + def on_val_end(self): # Callback runs on val end if self.wandb or self.clearml: @@ -188,6 +236,9 @@ def on_fit_epoch_end(self, vals, epoch, best_fitness, fi): self.clearml.current_epoch_logged_images = set() # reset epoch image limit self.clearml.current_epoch += 1 + if self.comet_logger: + self.comet_logger.on_fit_epoch_end(x, epoch=epoch) + def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): # Callback runs on model save event if (epoch + 1) % self.opt.save_period == 0 and not final_epoch and self.opt.save_period != -1: @@ -198,6 +249,9 @@ def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): model_name='Latest Model', auto_delete_file=False) + if self.comet_logger: + self.comet_logger.on_model_save(last, epoch, final_epoch, best_fitness, fi) + def on_train_end(self, last, best, epoch, results): # Callback runs on training end, i.e. saving best model if self.plots: @@ -224,6 +278,10 @@ def on_train_end(self, last, best, epoch, results): if self.clearml and not self.opt.evolve: self.clearml.task.update_output_model(model_path=str(best if best.exists() else last), name='Best Model') + if self.comet_logger: + final_results = dict(zip(self.keys[3:10], results)) + self.comet_logger.on_train_end(files, self.save_dir, last, best, epoch, final_results) + def on_params_update(self, params: dict): # Update hyperparams or configs of the experiment if self.wandb: From 57ce76cb5e05da3b28a0c51d6de8808d96e4d1f8 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 05:21:17 +0000 Subject: [PATCH 02/41] add comet logger --- utils/loggers/comet/__init__.py | 466 +++++++++++++++++++++++++++++ utils/loggers/comet/comet_utils.py | 145 +++++++++ 2 files changed, 611 insertions(+) create mode 100644 utils/loggers/comet/__init__.py create mode 100644 utils/loggers/comet/comet_utils.py diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py new file mode 100644 index 000000000000..685fc8107d67 --- /dev/null +++ b/utils/loggers/comet/__init__.py @@ -0,0 +1,466 @@ +import glob +import json +import logging +import os +from pathlib import Path + +logger = logging.getLogger(__name__) + +try: + import comet_ml + + # Project Configuration + config = comet_ml.config.get_config() + COMET_PROJECT_NAME = config.get_string(os.getenv("COMET_PROJECT_NAME"), "comet.project_name", default="yolov5") +except (ModuleNotFoundError, ImportError): + comet_ml = None + COMET_PROJECT_NAME = None + +import torch +import torchvision.transforms as T +import yaml + +from utils.dataloaders import img2label_paths +from utils.general import check_dataset, scale_coords, xywh2xyxy +from utils.metrics import ConfusionMatrix, box_iou + +COMET_MODE = os.getenv("COMET_MODE", "online") + +# Model Saving Settings +COMET_SAVE_MODEL = os.getenv("COMET_SAVE_MODEL", "false").lower() == "true" +COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5") +COMET_OVERWRITE_CHECKPOINTS = (os.getenv("COMET_OVERWRITE_CHECKPOINTS", "false").lower() == "true") + +# Dataset Artifact Settings +COMET_UPLOAD_DATASET = os.getenv("COMET_UPLOAD_DATASET", "false").lower() == "true" + +# Evaluation Settings +COMET_LOG_CONFUSION_MATRIX = (os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true") +COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "false").lower() == "true" +COMET_MAX_IMAGE_UPLOADS = os.getenv("COMET_MAX_IMAGE_UPLOADS", 100) + +# Confusion Matrix Settings +CONF_THRES = os.getenv("CONF_THRES", 0.001) +IOU_THRES = os.getenv("IOU_THRES", 0.6) + +# Batch Logging Settings +COMET_LOG_BATCH_METRICS = (os.getenv("COMET_LOG_BATCH_METRICS", "false").lower() == "true") +COMET_BATCH_LOGGING_INTERVAL = os.getenv("COMET_BATCH_LOGGING_INTERVAL", 1) +COMET_PREDICTION_LOGGING_INTERVAL = os.getenv("COMET_PREDICTION_LOGGING_INTERVAL", 1) + +RANK = int(os.getenv("RANK", -1)) + +to_pil = T.ToPILImage() + + +class CometLogger: + """Log metrics, parameters, source code, models and much more + with Comet + """ + + def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwargs) -> None: + self.job_type = job_type + self.opt = opt + self.hyp = hyp + + # Comet Flags + self.comet_mode = self.opt.comet_mode if self.opt.comet_mode else COMET_MODE + + self.save_model = (opt.comet_save_model if opt.comet_save_model else COMET_SAVE_MODEL) + self.model_name = (opt.comet_model_name if opt.comet_model_name else COMET_MODEL_NAME) + self.overwrite_checkpoints = (opt.comet_overwrite_checkpoints + if opt.comet_overwrite_checkpoints else COMET_OVERWRITE_CHECKPOINTS) + + # Batch Logging Settings + self.log_batch_metrics = (opt.comet_log_batch_metrics + if opt.comet_log_batch_metrics else COMET_LOG_BATCH_METRICS) + self.comet_log_batch_interval = (opt.comet_log_batch_interval + if opt.comet_log_batch_interval else COMET_BATCH_LOGGING_INTERVAL) + + # Dataset Artifact Settings + self.upload_dataset = (self.opt.comet_upload_dataset if self.opt.comet_upload_dataset else COMET_UPLOAD_DATASET) + self.resume = self.opt.resume + + # Default parameters to pass to Experiment objects + self.default_experiment_kwargs = { + "log_code": False, + "log_env_gpu": True, + "log_env_cpu": True, + "project_name": COMET_PROJECT_NAME,} + self.default_experiment_kwargs.update(experiment_kwargs) + self.experiment = self._get_experiment(self.comet_mode, run_id) + + if self.opt.comet_artifact: + self.data_dict = self.download_dataset_artifact(self.opt.comet_artifact) + + else: + self.data_dict = check_dataset(self.opt.data) + + self.class_names = self.data_dict["names"] + self.num_classes = self.data_dict["nc"] + + self.logged_images_count = 0 + self.max_images = (self.opt.comet_max_image_uploads + if self.opt.comet_max_image_uploads else COMET_MAX_IMAGE_UPLOADS) + if self.experiment is not None: + if run_id is None: + self.experiment.log_other("Created from", "YOLOv5") + + if not isinstance(self.experiment, comet_ml.OfflineExperiment): + workspace, project_name, experiment_id = self.experiment.url.split("/")[-3:] + self.experiment.log_other( + "Run Path", + f"{workspace}/{project_name}/{experiment_id}", + ) + self.log_parameters(vars(opt)) + self.log_parameters(self.opt.hyp) + self.log_asset_data( + self.opt.hyp, + name="hyperparameters.json", + metadata={"type": "hyp-config-file"}, + ) + self.log_asset( + f"{self.opt.save_dir}/opt.yaml", + metadata={"type": "opt-config-file"}, + ) + if not self.opt.comet_artifact: + self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) + + self.comet_log_confusion_matrix = (self.opt.comet_log_confusion_matrix + if self.opt.comet_log_confusion_matrix else COMET_LOG_CONFUSION_MATRIX) + if self.comet_log_confusion_matrix: + if hasattr(self.opt, "conf_thres"): + self.conf_thres = self.opt.conf_thres + else: + self.conf_thres = CONF_THRES + if hasattr(self.opt, "iou_thres"): + self.iou_thres = self.opt.iou_thres + else: + self.iou_thres = IOU_THRES + + self.comet_log_predictions = (self.opt.comet_log_predictions + if self.opt.comet_log_predictions else COMET_LOG_PREDICTIONS) + self.comet_log_prediction_interval = (opt.comet_log_prediction_interval if opt.comet_log_prediction_interval + else COMET_PREDICTION_LOGGING_INTERVAL) + if self.comet_log_predictions: + self.metadata_dict = {} + + # Check if running the Experiment with the Comet Optimizer + if hasattr(self.opt, "comet_optimizer_id"): + self.experiment.log_other("optimizer_id", self.opt.comet_optimizer_id) + self.experiment.log_other("optimizer_objective", self.opt.comet_optimizer_objective) + self.experiment.log_other("optimizer_metric", self.opt.comet_optimizer_metric) + self.experiment.log_other("optimizer_parameters", json.dumps(self.hyp)) + + def _get_experiment(self, mode, experiment_id=None): + if mode == "offline": + if experiment_id is not None: + return comet_ml.ExistingOfflineExperiment( + previous_experiment=experiment_id, + **self.default_experiment_kwargs, + ) + + return comet_ml.OfflineExperiment(**self.default_experiment_kwargs,) + + else: + try: + if experiment_id is not None: + return comet_ml.ExistingExperiment( + previous_experiment=experiment_id, + **self.default_experiment_kwargs, + ) + + return comet_ml.Experiment(**self.default_experiment_kwargs,) + + except ValueError as e: + logger.warning("COMET WARNING: " + "Comet credentials have not been set. " + "Comet will default to offline logging. " + "Please set your credentials to enable online logging.") + logger.exceptiom(e) + return self._get_experiment("offline", experiment_id) + + return + + def log_metrics(self, log_dict, **kwargs): + self.experiment.log_metrics(log_dict, **kwargs) + + def log_parameters(self, log_dict, **kwargs): + self.experiment.log_parameters(log_dict, **kwargs) + + def log_asset(self, asset_path, **kwargs): + self.experiment.log_asset(asset_path, **kwargs) + + def log_asset_data(self, asset, **kwargs): + self.experiment.log_asset_data(asset, **kwargs) + + def log_image(self, img, **kwargs): + self.experiment.log_image(img, **kwargs) + + def log_model(self, path, opt, epoch, fitness_score, best_model=False): + if not self.save_model: + return + + model_metadata = { + "fitness_score": fitness_score[-1], + "epochs_trained": epoch + 1, + "save_period": opt.save_period, + "total_epochs": opt.epochs,} + + if opt.comet_checkpoint_filename == "all": + model_path = str(path) + model_files = glob.glob(f"{path}/*.pt") + + else: + model_files = [str(path) + f"/{opt.comet_checkpoint_filename}"] + + for model_path in model_files: + if not self.overwrite_checkpoints: + name = f"{Path(model_path).stem}_epoch_{epoch}.pt" + else: + name = Path(model_path).name + + self.experiment.log_model( + self.model_name, + file_or_folder=model_path, + file_name=name, + metadata=model_metadata, + overwrite=self.overwrite_checkpoints, + ) + + def log_predictions(self, image, labelsn, path, shape, predn): + if self.logged_images_count >= self.max_images: + return + + detections = predn[predn[:, 4] > self.conf_thres] + iou = box_iou(labelsn[:, 1:], detections[:, :4]) + mask, _ = torch.where(iou > self.iou_thres) + if len(mask) == 0: + return + + filtered_detections = detections[mask] + filtered_labels = labelsn[mask] + + processed_image = (image * 255).to(torch.uint8) + + image_id = path.split("/")[-1].split(".")[0] + image_name = f"{image_id}_curr_epoch_{self.experiment.curr_epoch}" + self.log_image(to_pil(processed_image), name=image_name) + + metadata = [] + for cls, *xyxy in filtered_labels.tolist(): + metadata.append({ + "label": f"{self.class_names[int(cls)]}-gt", + "score": 100, + "box": { + "x": xyxy[0], + "y": xyxy[1], + "x2": xyxy[2], + "y2": xyxy[3]},}) + for *xyxy, conf, cls in filtered_detections.tolist(): + metadata.append({ + "label": f"{self.class_names[int(cls)]}", + "score": conf * 100, + "box": { + "x": xyxy[0], + "y": xyxy[1], + "x2": xyxy[2], + "y2": xyxy[3]},}) + + self.metadata_dict[image_name] = metadata + self.logged_images_count += 1 + + return + + def preprocess_prediction(self, image, labels, shape, pred): + nl, _ = labels.shape[0], pred.shape[0] + + # Predictions + if self.opt.single_cls: + pred[:, 5] = 0 + + predn = pred.clone() + scale_coords(image.shape[1:], predn[:, :4], shape[0], shape[1]) + + labelsn = None + if nl: + tbox = xywh2xyxy(labels[:, 1:5]) # target boxes + scale_coords(image.shape[1:], tbox, shape[0], shape[1]) # native-space labels + labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels + scale_coords(image.shape[1:], predn[:, :4], shape[0], shape[1]) # native-space pred + + return predn, labelsn + + def add_assets_to_artifact(self, artifact, path, asset_path, split): + img_paths = sorted(glob.glob(f"{asset_path}/*")) + label_paths = img2label_paths(img_paths) + + for image_file, label_file in zip(img_paths, label_paths): + image_logical_path, label_logical_path = map(lambda x: x.replace(f"{path}/", ""), [image_file, label_file]) + artifact.add(image_file, logical_path=image_logical_path, metadata={"split": split}) + artifact.add(label_file, logical_path=label_logical_path, metadata={"split": split}) + return artifact + + def upload_dataset_artifact(self): + dataset_name = self.data_dict.get("dataset_name", "yolov5-dataset") + path = self.data_dict["path"] + + metadata = self.data_dict.copy() + for key in ["train", "val", "test"]: + split_path = metadata.get(key) + if split_path is not None: + metadata[key] = split_path.replace(path, "") + + artifact = comet_ml.Artifact(name=dataset_name, artifact_type="dataset", metadata=metadata) + for key in metadata.keys(): + if key in ["train", "val", "test"]: + if isinstance(self.upload_dataset, str) and (key != self.upload_dataset): + continue + + asset_path = self.data_dict.get(key) + if asset_path is not None: + artifact = self.add_assets_to_artifact(artifact, path, asset_path, key) + + self.experiment.log_artifact(artifact) + + return + + def download_dataset_artifact(self, artifact_path): + logged_artifact = self.experiment.get_artifact(artifact_path) + logged_artifact.download(self.opt.save_dir) + + metadata = logged_artifact.metadata + data_dict = metadata.copy() + data_dict["path"] = self.opt.save_dir + data_dict["names"] = {int(k): v for k, v in metadata.get("names").items()} + + data_dict = self.update_data_paths(data_dict) + return data_dict + + def update_data_paths(self, data_dict): + path = data_dict.get("path", "") + + for split in ["train", "val", "test"]: + if data_dict.get(split): + split_path = data_dict.get(split) + data_dict[split] = (f"{path}/{split_path}" if isinstance(split, str) else [ + f"{path}/{x}" for x in split_path]) + + return data_dict + + def on_pretrain_routine_end(self, paths): + if self.opt.resume: + return + + for path in paths: + self.log_asset(str(path)) + + if self.upload_dataset: + if not self.resume: + self.upload_dataset_artifact() + + return + + def on_train_start(self): + self.log_parameters(self.hyp) + + def on_train_epoch_start(self): + return + + def on_train_epoch_end(self, epoch): + self.experiment.curr_epoch = epoch + + return + + def on_train_batch_start(self): + return + + def on_train_batch_end(self, log_dict, step): + self.experiment.curr_step = step + if self.log_batch_metrics and (step % self.comet_log_batch_interval == 0): + self.log_metrics(log_dict, step=step) + + return + + def on_train_end(self, files, save_dir, last, best, epoch, results): + if self.comet_log_predictions: + curr_epoch = self.experiment.curr_epoch + self.experiment.log_asset_data(self.metadata_dict, "image-metadata.json", epoch=curr_epoch) + + for f in files: + self.log_asset(f, metadata={"epoch": epoch}) + self.log_asset(f"{save_dir}/results.csv", metadata={"epoch": epoch}) + + if not self.opt.evolve: + model_path = str(best if best.exists() else last) + if not self.overwrite_checkpoints: + name = name = f"{Path(model_path).stem}_epoch_{epoch}.pt" + else: + name = Path(model_path).name + if self.save_model: + self.experiment.log_model( + self.model_name, + file_or_folder=model_path, + file_name=name, + overwrite=self.overwrite_checkpoints, + ) + + # Check if running Experiment with Comet Optimizer + if hasattr(self.opt, 'comet_optimizer_id'): + metric = results.get(self.opt.comet_optimizer_metric) + self.experiment.log_other('optimizer_metric_value', metric) + + self.finish_run() + + def on_val_start(self): + self.confmat = ConfusionMatrix(nc=self.num_classes, conf=self.conf_thres, iou_thres=self.iou_thres) + + return + + def on_val_batch_start(self): + return + + def on_val_batch_end(self, batch_i, images, targets, paths, shapes, outputs): + for si, pred in enumerate(outputs): + if len(pred) == 0: + continue + + image = images[si] + labels = targets[targets[:, 0] == si, 1:] + shape = shapes[si] + path = paths[si] + + predn, labelsn = self.preprocess_prediction(image, labels, shape, pred) + if labelsn is not None: + if self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0): + self.log_predictions(image, labelsn, path, shape, predn) + + if self.comet_log_confusion_matrix: + self.confmat.process_batch(predn, labelsn) + + return + + def on_fit_epoch_end(self, result, epoch): + self.log_metrics(result, epoch=epoch) + + if self.comet_log_confusion_matrix: + class_names = list(self.class_names.values()) + class_names.append("background-FN") + + num_classes = len(class_names) + + self.experiment.log_confusion_matrix( + matrix=self.confmat.matrix, + max_categories=num_classes, + labels=class_names, + epoch=epoch, + file_name=f"confusion-matrix-epoch-{epoch}.json", + ) + + def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): + if ((epoch + 1) % self.opt.save_period == 0 and not final_epoch) and self.opt.save_period != -1: + self.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi) + + def finish_run(self): + if self.experiment is not None: + self.experiment.end() diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py new file mode 100644 index 000000000000..f3dc0feeb629 --- /dev/null +++ b/utils/loggers/comet/comet_utils.py @@ -0,0 +1,145 @@ +import logging +import os +from urllib.parse import urlparse + +try: + import comet_ml +except (ModuleNotFoundError, ImportError): + comet_ml = None + +import yaml + +logger = logging.getLogger(__name__) + +COMET_PREFIX = "comet://" +COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5") + + +def download_model_checkpoint(opt, experiment): + model_dir = f"{opt.project}/{experiment.name}" + os.makedirs(model_dir, exist_ok=True) + + model_name = opt.comet_model_name if opt.comet_model_name else COMET_MODEL_NAME + model_asset_list = experiment.get_model_asset_list(model_name) + + if len(model_asset_list) == 0: + logger.error(f"COMET ERROR: No checkpoints found for model name : {model_name}") + return + + model_asset_list = sorted( + model_asset_list, + key=lambda x: x["step"], + reverse=True, + ) + + checkpoint_filename = opt.comet_checkpoint_filename + logged_checkpoint_map = {asset["fileName"]: asset["assetId"] for asset in model_asset_list} + asset_id = logged_checkpoint_map.get(checkpoint_filename) + + try: + if asset_id is None: + # Fetch latest checkpoint + asset_id = model_asset_list[0]["assetId"] + asset_filename = model_asset_list[0]["fileName"] + logger.info(f"COMET INFO: Checkpoint {checkpoint_filename} not found." + f"Defaulting to latest checkpoint {asset_filename}") + + else: + asset_filename = checkpoint_filename + + model_binary = experiment.get_asset(asset_id, return_type="binary", stream=False) + model_download_path = f"{model_dir}/{asset_filename}" + with open(model_download_path, "wb") as f: + f.write(model_binary) + + opt.weights = model_download_path + + except Exception as e: + logger.warning("COMET WARNING: Unable to download checkpoint from Comet") + logger.exception(e) + + +def set_opt_parameters(opt, experiment): + """Update the opts Namespace with parameters + from Comet's ExistingExperiment when resuming a run + + Args: + opt (argparse.Namespace): Namespace of command line options + experiment (comet_ml.APIExperiment): Comet API Experiment object + """ + asset_list = experiment.get_asset_list() + resume_string = opt.resume + for asset in asset_list: + if asset["fileName"] == "opt.yaml": + asset_id = asset["assetId"] + asset_binary = experiment.get_asset(asset_id, return_type="binary", stream=False) + opt_dict = yaml.safe_load(asset_binary) + for key, value in opt_dict.items(): + setattr(opt, key, value) + opt.resume = resume_string + + # Save hyperparamers to YAML file + # Necessary to pass checks in training script + save_dir = f"{opt.project}/{experiment.name}" + os.makedirs(save_dir, exist_ok=True) + + hyp_yaml_path = f"{save_dir}/hyp.yaml" + with open(hyp_yaml_path, "w") as f: + yaml.dump(opt.hyp, f) + opt.hyp = hyp_yaml_path + + +def check_comet_weights(opt): + """Downloads model weights from Comet and updates the + weights path to point to saved weights location + + Args: + opt (argparse.Namespace): Command Line arguments passed + to YOLOv5 training script + + Returns: + None/bool: Return True if weights are successfully downloaded + else return None + """ + if comet_ml is None: + return + + if isinstance(opt.weights, str): + if opt.weights.startswith(COMET_PREFIX): + api = comet_ml.API() + resource = urlparse(opt.weights) + experiment_path = f"{resource.netloc}{resource.path}" + experiment = api.get(experiment_path) + download_model_checkpoint(opt, experiment) + return True + + return None + + +def check_comet_resume(opt): + """Restores run parameters to its original state based on the model checkpoint + and logged Experiment parameters. + + Args: + opt (argparse.Namespace): Command Line arguments passed + to YOLOv5 training script + + Returns: + None/bool: Return True if the run is restored successfully + else return None + """ + if comet_ml is None: + return + + if isinstance(opt.resume, str): + if opt.resume.startswith(COMET_PREFIX): + api = comet_ml.API() + resource = urlparse(opt.resume) + experiment_path = f"{resource.netloc}{resource.path}" + experiment = api.get(experiment_path) + set_opt_parameters(opt, experiment) + download_model_checkpoint(opt, experiment) + + return True + + return None From fa6610e761d674db2a837f4c20c86d0274928ea4 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 05:35:44 +0000 Subject: [PATCH 03/41] add support for updated parameters --- utils/loggers/__init__.py | 2 ++ utils/loggers/comet/__init__.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 61ff34925534..0f1bc71c85fe 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -286,6 +286,8 @@ def on_params_update(self, params: dict): # Update hyperparams or configs of the experiment if self.wandb: self.wandb.wandb_run.config.update(params, allow_val_change=True) + if self.comet_logger: + self.comet_logger.on_params_update(params) class GenericLogger: diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 685fc8107d67..94dacb8e0c16 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -461,6 +461,9 @@ def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): if ((epoch + 1) % self.opt.save_period == 0 and not final_epoch) and self.opt.save_period != -1: self.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi) + def on_params_update(self, params): + self.log_parameters(params) + def finish_run(self): if self.experiment is not None: self.experiment.end() From 1fce3df92ca7c55dd30a1e28e3dc813959e782e7 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 05:58:11 +0000 Subject: [PATCH 04/41] clean up offline logger creation --- train.py | 52 +++++++++++++++++++++++++++++++-- utils/loggers/__init__.py | 2 +- utils/loggers/comet/__init__.py | 5 ++-- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/train.py b/train.py index 0cd4a7f065a6..4110260569d8 100644 --- a/train.py +++ b/train.py @@ -52,6 +52,7 @@ init_seeds, intersect_dicts, labels_to_class_weights, labels_to_image_weights, methods, one_cycle, print_args, print_mutation, strip_optimizer, yaml_save) from utils.loggers import Loggers +from utils.loggers.comet.comet_utils import check_comet_resume, check_comet_weights from utils.loggers.wandb.wandb_utils import check_wandb_resume from utils.loss import ComputeLoss from utils.metrics import fitness @@ -97,7 +98,8 @@ def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictio data_dict = loggers.wandb.data_dict if resume: weights, epochs, hyp, batch_size = opt.weights, opt.epochs, opt.hyp, opt.batch_size - + if loggers.comet_logger: + data_dict = loggers.comet_logger.data_dict # Register actions for k in methods(loggers): callbacks.register_action(k, callback=getattr(loggers, k)) @@ -472,6 +474,49 @@ def parse_opt(known=False): parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval') parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') + # Comet Arguments + parser.add_argument("--comet_mode", type=str, help="Comet: Set whether to run Comet in online or offline mode.") + parser.add_argument("--comet_save_model", action="store_true", help="Comet: Set to save model checkpoints.") + parser.add_argument("--comet_model_name", type=str, help="Comet: Set the name for the saved model.") + parser.add_argument("--comet_overwrite_checkpoints", + action="store_true", + help="Comet: Overwrite existing model checkpoints.") + parser.add_argument("--comet_checkpoint_filename", + nargs="?", + type=str, + default="best.pt", + help=("Comet: Name of the checkpoint file to save to Comet." + "Set to 'all' to log all checkpoints.")) + parser.add_argument("--comet_log_batch_metrics", + action="store_true", + help="Comet: Set to log batch level training metrics.") + parser.add_argument("--comet_log_batch_interval", + type=int, + default=1, + help="Comet: Logging frequency for batch level training metrics.") + parser.add_argument("--comet_log_prediction_interval", + type=int, + default=1, + help=("Comet: How often to log predictions." + "Applied at batch level.")) + parser.add_argument("--comet_log_confusion_matrix", + action="store_true", + help="Comet: Log a Confusion Matrix for the validation dataset.") + parser.add_argument("--comet_log_predictions", + action="store_true", + help="Comet: Log Predictions on Images from the Validation Set") + parser.add_argument("--comet_max_image_uploads", + type=int, + default=100, + help="Comet: Maximum number of images to log to Comet.") + parser.add_argument("--comet_upload_dataset", + nargs="?", + const=True, + default=False, + help=("Comet: Upload Dataset to Comet as an Artifact." + "Set to 'train', 'val' or 'test' to upload a single dataset.")) + parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") + return parser.parse_known_args()[0] if known else parser.parse_args() @@ -482,8 +527,11 @@ def main(opt, callbacks=Callbacks()): check_git_status() check_requirements() + check_comet_weights(opt) + # Resume - if opt.resume and not (check_wandb_resume(opt) or opt.evolve): # resume from specified or most recent last.pt + if opt.resume and not check_wandb_resume(opt) or check_comet_resume( + opt) or opt.evolve: # resume from specified or most recent last.pt last = Path(check_file(opt.resume) if isinstance(opt.resume, str) else get_latest_run()) opt_yaml = last.parent.parent / 'opt.yaml' # train options yaml opt_data = opt.data # original dataset diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 0f1bc71c85fe..2d8c3c8ab5aa 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -17,7 +17,7 @@ from utils.plots import plot_images, plot_labels, plot_results from utils.torch_utils import de_parallel -LOGGERS = ('csv', 'tb', 'wandb', 'clearml', 'comet_ml') # *.csv, TensorBoard, Weights & Biases, ClearML +LOGGERS = ('csv', 'tb', 'wandb', 'clearml', 'comet') # *.csv, TensorBoard, Weights & Biases, ClearML RANK = int(os.getenv('RANK', -1)) try: diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 94dacb8e0c16..bfd840532570 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -170,14 +170,13 @@ def _get_experiment(self, mode, experiment_id=None): **self.default_experiment_kwargs, ) - return comet_ml.Experiment(**self.default_experiment_kwargs,) + return comet_ml.Experiment(**self.default_experiment_kwargs) - except ValueError as e: + except ValueError: logger.warning("COMET WARNING: " "Comet credentials have not been set. " "Comet will default to offline logging. " "Please set your credentials to enable online logging.") - logger.exceptiom(e) return self._get_experiment("offline", experiment_id) return From 751dcc3ef667b1d1ff5840d6760d6c5f27c00758 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 06:23:35 +0000 Subject: [PATCH 05/41] update callback args for comet logger --- train.py | 2 +- val.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 4110260569d8..7c6811dc0338 100644 --- a/train.py +++ b/train.py @@ -333,7 +333,7 @@ def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictio mem = f'{torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0:.3g}G' # (GB) pbar.set_description(('%11s' * 2 + '%11.4g' * 5) % (f'{epoch}/{epochs - 1}', mem, *mloss, targets.shape[0], imgs.shape[-1])) - callbacks.run('on_train_batch_end', model, ni, imgs, targets, paths) + callbacks.run('on_train_batch_end', model, ni, imgs, targets, paths, list(mloss)) if callbacks.stop_training: return # end batch ------------------------------------------------------------------------------------------------ diff --git a/val.py b/val.py index 58b9c9e1bec0..fdb4e5ec8853 100644 --- a/val.py +++ b/val.py @@ -259,7 +259,7 @@ def run( plot_images(im, targets, paths, save_dir / f'val_batch{batch_i}_labels.jpg', names) # labels plot_images(im, output_to_target(out), paths, save_dir / f'val_batch{batch_i}_pred.jpg', names) # pred - callbacks.run('on_val_batch_end') + callbacks.run('on_val_batch_end', batch_i, im, targets, paths, shapes, out) # Compute metrics stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy From 2e104544a9cc68c7e369b699476f241be4e74017 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 06:24:22 +0000 Subject: [PATCH 06/41] add comet optimizer --- utils/loggers/comet/hpo.py | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 utils/loggers/comet/hpo.py diff --git a/utils/loggers/comet/hpo.py b/utils/loggers/comet/hpo.py new file mode 100644 index 000000000000..01c2d1fffb96 --- /dev/null +++ b/utils/loggers/comet/hpo.py @@ -0,0 +1,151 @@ +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +import comet_ml + +logger = logging.getLogger(__name__) + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[3] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + +from train import parse_opt, train +from utils.callbacks import Callbacks +from utils.general import increment_path +from utils.torch_utils import select_device + +# Project Configuration +config = comet_ml.config.get_config() +COMET_PROJECT_NAME = config.get_string(os.getenv("COMET_PROJECT_NAME"), "comet.project_name", default="yolov5") + + +def get_args(known=False): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--hyp', type=str, default=ROOT / 'data/hyps/hyp.scratch-low.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=300, help='total training epochs') + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs, -1 for autobatch') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--noval', action='store_true', help='only validate final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable AutoAnchor') + parser.add_argument('--noplots', action='store_true', help='save no plot files') + parser.add_argument('--evolve', type=int, nargs='?', const=300, help='evolve hyperparameters for x generations') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache', type=str, nargs='?', const='ram', help='--cache images in "ram" (default) or "disk"') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--optimizer', type=str, choices=['SGD', 'Adam', 'AdamW'], default='SGD', help='optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--project', default=ROOT / 'runs/train', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--cos-lr', action='store_true', help='cosine LR scheduler') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--patience', type=int, default=100, help='EarlyStopping patience (epochs without improvement)') + parser.add_argument('--freeze', nargs='+', type=int, default=[0], help='Freeze layers: backbone=10, first3=0 1 2') + parser.add_argument('--save-period', type=int, default=-1, help='Save checkpoint every x epochs (disabled if < 1)') + parser.add_argument('--seed', type=int, default=0, help='Global training seed') + parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify') + + # Weights & Biases arguments + parser.add_argument('--entity', default=None, help='W&B: Entity') + parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='W&B: Upload data, "val" option') + parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval') + parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') + + # Comet Arguments + parser.add_argument("--comet_mode", type=str, help="Comet: Set whether to run Comet in online or offline mode.") + parser.add_argument("--comet_save_model", action="store_true", help="Comet: Set to save model checkpoints.") + parser.add_argument("--comet_model_name", type=str, help="Comet: Set the name for the saved model.") + parser.add_argument("--comet_overwrite_checkpoints", + action="store_true", + help="Comet: Overwrite existing model checkpoints.") + parser.add_argument("--comet_checkpoint_filename", + nargs="?", + type=str, + default="best.pt", + help=("Comet: Name of the checkpoint file to save to Comet." + "Set to 'all' to log all checkpoints.")) + parser.add_argument("--comet_log_batch_metrics", + action="store_true", + help="Comet: Set to log batch level training metrics.") + parser.add_argument("--comet_log_batch_interval", + type=int, + default=1, + help="Comet: Logging frequency for batch level training metrics.") + parser.add_argument("--comet_log_prediction_interval", + type=int, + default=1, + help=("Comet: How often to log predictions." + "Applied at batch level.")) + parser.add_argument("--comet_log_confusion_matrix", + action="store_true", + help="Comet: Log a Confusion Matrix for the validation dataset.") + parser.add_argument("--comet_log_predictions", + action="store_true", + help="Comet: Log Predictions on Images from the Validation Set") + parser.add_argument("--comet_max_image_uploads", + type=int, + default=100, + help="Comet: Maximum number of images to log to Comet.") + parser.add_argument("--comet_upload_dataset", + nargs="?", + const=True, + default=False, + help=("Comet: Upload Dataset to Comet as an Artifact." + "Set to 'train', 'val' or 'test' to upload a single dataset.")) + parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") + + return parser.parse_known_args()[0] if known else parser.parse_args() + + +def run(parameters, opt): + hyp_dict = {k: v for k, v in parameters.items() if k not in ["epochs", "batch_size"]} + + opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve)) + opt.batch_size = parameters.get("batch_size") + opt.epochs = parameters.get("epochs") + + device = select_device(opt.device, batch_size=opt.batch_size) + train(hyp_dict, opt, device, callbacks=Callbacks()) + + +if __name__ == "__main__": + opt = get_args(known=True) + + opt.weights = str(opt.weights) + opt.cfg = str(opt.cfg) + opt.data = str(opt.data) + opt.project = str(opt.project) + + optimizer_id = os.getenv("COMET_OPTIMIZER_ID") + if optimizer_id is None: + with open(opt.comet_optimizer_config) as f: + optimizer_config = json.load(f) + optimizer = comet_ml.Optimizer(optimizer_config) + else: + optimizer = comet_ml.Optimizer(optimizer_id) + + opt.comet_optimizer_id = optimizer.id + status = optimizer.status() + + opt.comet_optimizer_objective = status["spec"]["objective"] + opt.comet_optimizer_metric = status["spec"]["metric"] + + logger.info("COMET INFO: Starting Hyperparameter Sweep") + for parameter in optimizer.get_parameters(): + run(parameter["parameters"], opt) From d08e10b8952a67a992bcef9b14c320eca410d90e Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 06:34:57 +0000 Subject: [PATCH 07/41] add optimizer config --- utils/loggers/comet/optimizer_config.json | 210 ++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 utils/loggers/comet/optimizer_config.json diff --git a/utils/loggers/comet/optimizer_config.json b/utils/loggers/comet/optimizer_config.json new file mode 100644 index 000000000000..d0b3857ff96f --- /dev/null +++ b/utils/loggers/comet/optimizer_config.json @@ -0,0 +1,210 @@ +{ + "algorithm": "random", + "parameters": { + "anchor_t": { + "type": "discrete", + "values": [ + 2, + 8 + ] + }, + "batch_size": { + "type": "discrete", + "values": [ + 16, + 32, + 64, + 128 + ] + }, + "box": { + "type": "discrete", + "values": [ + 0.02, + 0.2 + ] + }, + "cls": { + "type": "discrete", + "values": [ + 0.2 + ] + }, + "cls_pw": { + "type": "discrete", + "values": [ + 0.5 + ] + }, + "copy_paste": { + "type": "discrete", + "values": [ + 1 + ] + }, + "degrees": { + "type": "discrete", + "values": [ + 0, + 45 + ] + }, + "epochs": { + "type": "discrete", + "values": [ + 5 + ] + }, + "fl_gamma": { + "type": "discrete", + "values": [ + 0 + ] + }, + "fliplr": { + "type": "discrete", + "values": [ + 0 + ] + }, + "flipud": { + "type": "discrete", + "values": [ + 0 + ] + }, + "hsv_h": { + "type": "discrete", + "values": [ + 0 + ] + }, + "hsv_s": { + "type": "discrete", + "values": [ + 0 + ] + }, + "hsv_v": { + "type": "discrete", + "values": [ + 0 + ] + }, + "iou_t": { + "type": "discrete", + "values": [ + 0.7 + ] + }, + "lr0": { + "type": "discrete", + "values": [ + 1e-05, + 0.1 + ] + }, + "lrf": { + "type": "discrete", + "values": [ + 0.01, + 1 + ] + }, + "mixup": { + "type": "discrete", + "values": [ + 1 + ] + }, + "momentum": { + "type": "discrete", + "values": [ + 0.6 + ] + }, + "mosaic": { + "type": "discrete", + "values": [ + 0 + ] + }, + "obj": { + "type": "discrete", + "values": [ + 0.2 + ] + }, + "obj_pw": { + "type": "discrete", + "values": [ + 0.5 + ] + }, + "optimizer": { + "type": "categorical", + "values": [ + "SGD", + "Adam", + "AdamW" + ] + }, + "perspective": { + "type": "discrete", + "values": [ + 0 + ] + }, + "scale": { + "type": "discrete", + "values": [ + 0 + ] + }, + "shear": { + "type": "discrete", + "values": [ + 0 + ] + }, + "translate": { + "type": "discrete", + "values": [ + 0 + ] + }, + "warmup_bias_lr": { + "type": "discrete", + "values": [ + 0, + 0.2 + ] + }, + "warmup_epochs": { + "type": "discrete", + "values": [ + 5 + ] + }, + "warmup_momentum": { + "type": "discrete", + "values": [ + 0, + 0.95 + ] + }, + "weight_decay": { + "type": "discrete", + "values": [ + 0, + 0.001 + ] + } + }, + "spec": { + "maxCombo": 0, + "metric": "metrics/mAP_0.5", + "objective": "maximize" + }, + "trials": 1 +} From 616e2dbb35360bc2dd2e119197f8f01f24875d1c Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 06:35:35 +0000 Subject: [PATCH 08/41] add comet README --- utils/loggers/comet/README.md | 368 ++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 utils/loggers/comet/README.md diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md new file mode 100644 index 000000000000..3263cfab1d9e --- /dev/null +++ b/utils/loggers/comet/README.md @@ -0,0 +1,368 @@ + + +# YOLOv5 with Comet + +This guide will cover how to use YOLOv5 with [Comet](https://www.comet.com/site/?ref=yolov5) + +# About Comet + +Comet builds tools that help data scientists, engineers, and team leaders accelerate and optimize machine learning and deep learning models. + +Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1?ref=yolov5)! + +Comet makes sure you never lose track of your work and makes it easy to share results and collaborate across teams of all sizes! + +# Getting Started + +## Install Comet + +```shell +pip install comet_ml +``` + +## Configure Comet Credentials + +There are two ways to configure Comet with YOLOv5. + +You can either set your credentials through enviroment variables + +**Environment Variables** + +```shell +export COMET_API_KEY= +export COMET_PROJECT_NAME= # This will default to 'yolov5' +``` + +Or create a `.comet.config` file in your working directory and set your credentials there. + +**Comet Configuration File** + +``` +[comet] +api_key= +project_name= # This will default to 'yolov5' +``` + +## Run the Training Script + +```shell +# Train YOLOv5s on COCO128 for 5 epochs +python train.py --img 640 --batch 16 --epochs 5 --data coco128.yaml --weights yolov5s.pt +``` + +That's it! Comet will automatically log your hyperparameters, command line arguments, training and valiation metrics. You can visualize and analyze your runs in the Comet UI + +yolo-ui + +# Try out an Example! +Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/353f9734261348b59b883660bcd62256?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5) + +Or better yet, try it out yourself in this Colab Notebook + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing) + +# Log automatically + +By default, Comet will log the following items + +## Metrics +- Box Loss, Object Loss, Classification Loss for the training and validation data +- mAP_0.5, mAP_0.5:0.95 metrics for the validation data. +- Precision and Recall for the validation data + +## Parameters + +- Model Hyperparameters +- All parameters passed through the command line options + +## Visualizations + +- Confusion Matrix of the model predictions on the validation data +- Plots for the PR and F1 curves across all classes +- Correlogram of the Class Labels + +# Configure Comet Logging + +Comet can be configured to log additional data either through command line flags passed to the training script +or through environment variables. + +Here is the full list of command line options for Comet + +```shell + --comet_mode COMET_MODE + Comet: Set whether to run Comet in online + or offline mode. + --comet_save_model Comet: Set to save model checkpoints. + --comet_model_name COMET_MODEL_NAME + Comet: Set the name for the saved model. + --comet_overwrite_checkpoints + Comet: Overwrite exsiting model + checkpoints. + --comet_checkpoint_filename [COMET_CHECKPOINT_FILENAME] + Comet: Name of the checkpoint file to save + to Comet.Set to 'all' to log all + checkpoints. + --comet_checkpoint_step COMET_CHECKPOINT_STEP + --comet_log_batch_metrics + Comet: Set to log batch level training + metrics. + --comet_log_batch_interval COMET_LOG_BATCH_INTERVAL + Comet: Logging frequency for batch level + training metrics. + --comet_log_prediction_interval COMET_LOG_PREDICTION_INTERVAL + Comet: How often to log predictions.Applied + at batch level. + --comet_log_confusion_matrix + Comet: Log a Confusion Matrix for the + validation dataset. + --comet_log_predictions + Comet: Log Predictions on Images from the + Validation Set + --comet_max_image_uploads COMET_MAX_IMAGE_UPLOADS + Comet: Maximum number of images to log to + Comet. + --comet_upload_dataset [COMET_UPLOAD_DATASET] + Comet: Upload Dataset to Comet as an + Artifact.Set to 'train', 'val' or 'test' to + upload a single dataset. + --comet_artifact COMET_ARTIFACT + Comet: Name of the Comet dataset Artifact + to download. +``` + +Let's take a look at these options. + +## Logging Checkpoints with Comet + +Logging Models to Comet is disabled by default. To enable it, pass the `comet_save_model` flag to the training script. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--save-period 1 \ +--comet_save_model +``` + +You have a few options when it comes to saving model checkpoints to Comet. Here's how you can change the checkpointing behavior. + +### Change which checkpoint file is logged +By default, Comet will save the `best.pt` checkpoint. To change which file gets saved, use the `comet_checkpoint_filename` argument. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--save-period 1 \ +--comet_save_model \ +--comet_checkpoint_filename "last.pt" # this defaults to "best.pt" +``` + +### Log all checkpoints files + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--save-period 1 \ +--comet_save_model \ +--comet_checkpoint_filename "all" +``` + +### Overwrite checkpoint files + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--save-period 1 \ +--comet_save_model \ +--comet_checkpoint_filename "best.pt" \ +--comet_overwrite_checkpoint +``` + +## Using a saved Checkpoint + +Comet will log a Run Path for every run that you can use to download model weights and resume runs. A run path is a string with the following format `comet:////` + +You can find the run path in the `Others` tab in your Comet Experiment. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" \ +``` + +By default, Comet will download the most recent checkpoint, this can be configured by specifying the checkpoint filename. You can specify a checkpoint for a particular epoch +by using the `comet_checkpoint_filename` flag. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" \ +--comet_checkpoint_filename "last_epoch_2.pt" +``` + +## Logging Model Predictions + +You can log model predictions and the associated images using `comet_log_predictions`. Predictions can be visualized using Comet's Object Detection Custom Panel + +Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1?ref=yolov5) + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_log_predictions +``` + +### Controlling the number of Prediction Images logged to Comet + +When logging predictions from YOLOv5, Comet will log the images associated with each set of predictions. By default a maximum of 100 validation images are logged. You can increase or decrease this number using the `comet_max_images` flag. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_log_predictions \ +--comet_max_image_uploads 200 +``` + +### Controlling the frequency of Prediction Images logged to Comet + +By default, every batch from the validation set is logged to Comet when logging predictions. This can be configured via the `comet_log_prediction_interval` flag. For example, setting this parameter to `2` will log every 2nd batch of data from the validation set. + +**Note:** The YOLOv5 validation dataloader will default to a batch size of 32, so you will have to set the logging frequency accordingly. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_log_predictions \ +--comet_log_prediction_interval 2 +``` + +## Uploading a Dataset to Comet Artifacts + +If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5), you can do so using the `comet_upload_dataset` flag. + +The dataset be organized in the way described in the [YOLOv5 documentation](https://docs.ultralytics.com/tutorials/train-custom-datasets/#3-organize-directories). The dataset config `yaml` file must follow the same format as that of the `coco128.yaml` file. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_upload_dataset +``` +You can find the uploaded dataset in the Artifacts tab in your Comet Workspace +artifact-1 + +You can preview the data directly in the Comet UI. +artifact-2 + +Artifacts are versioned and also support adding metadata about the dataset. Comet will automatically log the metadata from your dataset `yaml` file +artifact-3 + +### Using a saved Artifact + +If you would like to use a dataset from Comet Artifacts, simply pass the `comet_artifact` flag to the training script, along with the artifact path. The artifact path is a string with the following format `/:` + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_artifact "examples/yolov5-dataset:latest" +``` + +Artifacts also allow you to track the lineage of data as it flows through your Experimentation workflow. Here you can see a graph that shows you all the experiments that have used your uploaded dataset. +artifact-4 + +## Resuming a Training Run + +If your training run is interrupted for any reason, e.g. disrupted internet connection, you can resume the run using the `resume` flag and the Comet Run Path. + +The Run Path has the following format `comet:////`. + +This will restore the run to its state before the interruption, which includes restoring the model from a checkpoint, restoring all hyperparameters and training arguments and downloading Comet dataset Artifacts if they were used in the original run. The resumed run will continue logging to the existing Experiment in the Comet UI + +```shell +python train.py \ +--resume "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" +``` + +## Hyperparameter Search with the Comet Optimizer + +YOLOv5 is also integrated with Comet's Optimizer, making is simple to visualie hyperparameter sweeps in the Comet UI. + +### Configuring an Optimizer Sweep + +To configure the Comet Optimizer, you will have to create a JSON file with the information about the sweep. An example file has been provided [here](optimizer_config.json) + +```shell +python utils/loggers/comet/hpo.py \ + --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" \ +``` + +The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional Comet related arguments to your sweep simply add them after + +```shell +python utils/loggers/comet/hpo.py \ + --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" \ + --comet_save_model \ + --comet_overwrite_checkpoints +``` + +### Running a Sweep in Parallel + +```shell +comet optimizer -j utils/loggers/comet/hpo.py \ + utils/loggers/comet/optimizer_config.json" +``` + +The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional Comet related arguments to your sweep simply add them after + +```shell +comet optimizer -j utils/loggers/comet/hpo.py \ + utils/loggers/comet/optimizer_config.json" \ + --comet_save_model \ + --comet_overwrite_checkpoints +``` + +### Visualizing Results + +Comet provides a number of ways to visualize the results of your sweep. Take a look at a [project with a completed sweep here](https://www.comet.com/examples/comet-example-yolov5/view/PrlArHGuuhDTKC1UuBmTtOSXD/panels?ref=yolov5) + +hyperparameter-yolo \ No newline at end of file From 680ef8bd913ad1a2a54b19213e5ef39a880715bb Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 06:37:53 +0000 Subject: [PATCH 09/41] update tutorial notebook with Comet section --- tutorial.ipynb | 1235 ++++++++++++++++++++++++++---------------------- 1 file changed, 659 insertions(+), 576 deletions(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index 12840063b1f1..9909c0c3b2fa 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -1,516 +1,160 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "YOLOv5 Tutorial", - "provenance": [], - "collapsed_sections": [], - "machine_shape": "hm", - "toc_visible": true + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "t6MPjfT5NrKQ" + }, + "source": [ + "
\n", + "\n", + " \n", + " \n", + "\n", + "\n", + "
\n", + " \"Open\n", + " \"Open\n", + "
\n", + "\n", + "This YOLOv5 🚀 notebook by Ultralytics presents simple train, validate and predict examples to help start your AI adventure.
See GitHub for community support or contact us for professional support.\n", + "\n", + "
" + ] }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" + { + "cell_type": "markdown", + "metadata": { + "id": "7mGmQbAO5pQb" + }, + "source": [ + "# Setup\n", + "\n", + "Clone GitHub [repository](https://github.com/ultralytics/yolov5), install [dependencies](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) and check PyTorch and GPU." + ] }, - "accelerator": "GPU", - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "9b8caa3522fc4cbab31e13b5dfc7808d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_574140e4c4bc48c9a171541a02cd0211", - "IPY_MODEL_35e03ce5090346c9ae602891470fc555", - "IPY_MODEL_c942c208e72d46568b476bb0f2d75496" - ], - "layout": "IPY_MODEL_65881db1db8a4e9c930fab9172d45143" - } - }, - "574140e4c4bc48c9a171541a02cd0211": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_60b913d755b34d638478e30705a2dde1", - "placeholder": "​", - "style": "IPY_MODEL_0856bea36ec148b68522ff9c9eb258d8", - "value": "100%" - } - }, - "35e03ce5090346c9ae602891470fc555": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_76879f6f2aa54637a7a07faeea2bd684", - "max": 818322941, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_0ace3934ec6f4d36a1b3a9e086390926", - "value": 818322941 - } + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "c942c208e72d46568b476bb0f2d75496": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d6b7a2243e0c4beca714d99dceec23d6", - "placeholder": "​", - "style": "IPY_MODEL_5966ba6e6f114d8c9d8d1d6b1bd4f4c7", - "value": " 780M/780M [02:19<00:00, 6.24MB/s]" - } + "id": "wbvMlHd_QwMG", + "outputId": "0f9ee467-cea4-48e8-9050-7a76ae1b6141", + "vscode": { + "languageId": "python" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n" + ] }, - "65881db1db8a4e9c930fab9172d45143": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setup complete ✅ (8 CPUs, 51.0 GB RAM, 37.4/166.8 GB disk)\n" + ] + } + ], + "source": [ + "!git clone https://github.com/ultralytics/yolov5 # clone\n", + "%cd yolov5\n", + "%pip install -qr requirements.txt # install\n", + "\n", + "import torch\n", + "import utils\n", + "display = utils.notebook_init() # checks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4JnkELT0cIJg" + }, + "source": [ + "# 1. Detect\n", + "\n", + "`detect.py` runs YOLOv5 inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and saving results to `runs/detect`. Example inference sources are:\n", + "\n", + "```shell\n", + "python detect.py --source 0 # webcam\n", + " img.jpg # image \n", + " vid.mp4 # video\n", + " path/ # directory\n", + " 'path/*.jpg' # glob\n", + " 'https://youtu.be/Zgi9g1ksQHc' # YouTube\n", + " 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "60b913d755b34d638478e30705a2dde1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0856bea36ec148b68522ff9c9eb258d8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "76879f6f2aa54637a7a07faeea2bd684": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0ace3934ec6f4d36a1b3a9e086390926": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "d6b7a2243e0c4beca714d99dceec23d6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5966ba6e6f114d8c9d8d1d6b1bd4f4c7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + "id": "zR9ZbuQCH7FX", + "outputId": "60647b99-e8d4-402c-f444-331bf6746da4", + "vscode": { + "languageId": "python" } - } - } - }, - "cells": [ + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mdetect: \u001b[0mweights=['yolov5s.pt'], source=data/images, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/detect, name=exp, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False\n", + "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", + "\n", + "Downloading https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt to yolov5s.pt...\n", + "100% 14.1M/14.1M [00:00<00:00, 27.8MB/s]\n", + "\n", + "Fusing layers... \n", + "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients\n", + "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 bus, 14.8ms\n", + "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 2 ties, 20.1ms\n", + "Speed: 0.6ms pre-process, 17.4ms inference, 21.6ms NMS per image at shape (1, 3, 640, 640)\n", + "Results saved to \u001b[1mruns/detect/exp\u001b[0m\n" + ] + } + ], + "source": [ + "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images\n", + "# display.Image(filename='runs/detect/exp/zidane.jpg', width=600)" + ] + }, { "cell_type": "markdown", "metadata": { - "id": "t6MPjfT5NrKQ" + "id": "hkAzDWJ7cWTr" }, "source": [ - "
\n", - "\n", - " \n", - " \n", - "\n", - "\n", - "
\n", - " \"Open\n", - " \"Open\n", - "
\n", - "\n", - "This YOLOv5 🚀 notebook by Ultralytics presents simple train, validate and predict examples to help start your AI adventure.
See GitHub for community support or contact us for professional support.\n", - "\n", - "
" + "        \n", + "" ] }, { "cell_type": "markdown", "metadata": { - "id": "7mGmQbAO5pQb" + "id": "0eq1SMWl6Sfn" }, "source": [ - "# Setup\n", - "\n", - "Clone GitHub [repository](https://github.com/ultralytics/yolov5), install [dependencies](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) and check PyTorch and GPU." + "# 2. Validate\n", + "Validate a model's accuracy on the [COCO](https://cocodataset.org/#home) dataset's `val` or `test` splits. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases). To show results by class use the `--verbose` flag." ] }, { "cell_type": "code", + "execution_count": 3, "metadata": { - "id": "wbvMlHd_QwMG", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "0f9ee467-cea4-48e8-9050-7a76ae1b6141" - }, - "source": [ - "!git clone https://github.com/ultralytics/yolov5 # clone\n", - "%cd yolov5\n", - "%pip install -qr requirements.txt # install\n", - "\n", - "import torch\n", - "import utils\n", - "display = utils.notebook_init() # checks" - ], - "execution_count": 1, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n" - ] - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Setup complete ✅ (8 CPUs, 51.0 GB RAM, 37.4/166.8 GB disk)\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4JnkELT0cIJg" - }, - "source": [ - "# 1. Detect\n", - "\n", - "`detect.py` runs YOLOv5 inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and saving results to `runs/detect`. Example inference sources are:\n", - "\n", - "```shell\n", - "python detect.py --source 0 # webcam\n", - " img.jpg # image \n", - " vid.mp4 # video\n", - " path/ # directory\n", - " 'path/*.jpg' # glob\n", - " 'https://youtu.be/Zgi9g1ksQHc' # YouTube\n", - " 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream\n", - "```" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "zR9ZbuQCH7FX", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "60647b99-e8d4-402c-f444-331bf6746da4" - }, - "source": [ - "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images\n", - "# display.Image(filename='runs/detect/exp/zidane.jpg', width=600)" - ], - "execution_count": 2, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\u001b[34m\u001b[1mdetect: \u001b[0mweights=['yolov5s.pt'], source=data/images, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/detect, name=exp, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False\n", - "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", - "\n", - "Downloading https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt to yolov5s.pt...\n", - "100% 14.1M/14.1M [00:00<00:00, 27.8MB/s]\n", - "\n", - "Fusing layers... \n", - "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients\n", - "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 bus, 14.8ms\n", - "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 2 ties, 20.1ms\n", - "Speed: 0.6ms pre-process, 17.4ms inference, 21.6ms NMS per image at shape (1, 3, 640, 640)\n", - "Results saved to \u001b[1mruns/detect/exp\u001b[0m\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hkAzDWJ7cWTr" - }, - "source": [ - "        \n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0eq1SMWl6Sfn" - }, - "source": [ - "# 2. Validate\n", - "Validate a model's accuracy on the [COCO](https://cocodataset.org/#home) dataset's `val` or `test` splits. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases). To show results by class use the `--verbose` flag." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "WQPtK1QYVaD_", "colab": { "base_uri": "https://localhost:8080/", "height": 49, @@ -528,49 +172,51 @@ "5966ba6e6f114d8c9d8d1d6b1bd4f4c7" ] }, - "outputId": "102dabed-bc31-42fe-9133-d9ce28a2c01e" + "id": "WQPtK1QYVaD_", + "outputId": "102dabed-bc31-42fe-9133-d9ce28a2c01e", + "vscode": { + "languageId": "python" + } }, - "source": [ - "# Download COCO val\n", - "torch.hub.download_url_to_file('https://ultralytics.com/assets/coco2017val.zip', 'tmp.zip') # download (780M - 5000 images)\n", - "!unzip -q tmp.zip -d ../datasets && rm tmp.zip # unzip" - ], - "execution_count": 3, "outputs": [ { - "output_type": "display_data", "data": { - "text/plain": [ - " 0%| | 0.00/780M [00:00\n", + "```\n", + "\n", + "**Run the YOLO training script**\n", + "```shell\n", + "python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt\n", + "```\n", + "\n", + "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/comet). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?ref=yolov5) \n", + "\n", + "Get started by trying out the Comet Colab Notebook:\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing)\n", + "\n", + "\"yolo-ui\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Lay2WsTjNJzP" + }, "source": [ "## ClearML Logging and Automation 🌟 NEW\n", "\n", @@ -873,10 +570,7 @@ "\n", "\n", "\"ClearML" - ], - "metadata": { - "id": "Lay2WsTjNJzP" - } + ] }, { "cell_type": "markdown", @@ -951,9 +645,14 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "GMusP4OAxFu6" + "id": "GMusP4OAxFu6", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "import torch\n", "\n", @@ -968,15 +667,18 @@ "\n", "# Results\n", "results.print() # or .show(), .save(), .crop(), .pandas(), etc." - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "FGH0ZjkGjejy" + "id": "FGH0ZjkGjejy", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "# YOLOv5 CI\n", "%%shell\n", @@ -994,29 +696,35 @@ "python models/tf.py --weights $m.pt # build TF model\n", "python models/yolo.py --cfg $m.yaml # build PyTorch model\n", "python export.py --weights $m.pt --img 64 --include torchscript # export" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "mcKoSIK2WSzj" + "id": "mcKoSIK2WSzj", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "# Reproduce\n", "for x in (f'yolov5{x}' for x in 'nsmlx'):\n", " !python val.py --weights {x}.pt --data coco.yaml --img 640 --task speed # speed\n", " !python val.py --weights {x}.pt --data coco.yaml --img 640 --conf 0.001 --iou 0.65 # mAP" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "gogI-kwi3Tye" + "id": "gogI-kwi3Tye", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "# Profile\n", "from utils.torch_utils import profile\n", @@ -1024,76 +732,451 @@ "m1 = lambda x: x * torch.sigmoid(x)\n", "m2 = torch.nn.SiLU()\n", "results = profile(input=torch.randn(16, 3, 640, 640), ops=[m1, m2], n=100)" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "BSgFCAcMbk1R" + "id": "BSgFCAcMbk1R", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "# VOC\n", "for b, m in zip([64, 64, 64, 32, 16], [f'yolov5{x}' for x in 'nsmlx']): # batch, model\n", " !python train.py --batch {b} --weights {m}.pt --data VOC.yaml --epochs 50 --img 512 --hyp hyp.VOC.yaml --project VOC --name {m} --cache" - ], - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UWGH7H6yakVl", + "vscode": { + "languageId": "python" + } + }, + "outputs": [], "source": [ "# Classification train\n", "for m in [*(f'yolov5{x}-cls.pt' for x in 'nsmlx'), 'resnet50.pt', 'resnet101.pt', 'efficientnet_b0.pt', 'efficientnet_b1.pt']:\n", " for d in 'mnist', 'fashion-mnist', 'cifar10', 'cifar100', 'imagenette160', 'imagenette320', 'imagenette', 'imagewoof160', 'imagewoof320', 'imagewoof':\n", " !python classify/train.py --model {m} --data {d} --epochs 10 --project YOLOv5-cls --name {m}-{d}" - ], - "metadata": { - "id": "UWGH7H6yakVl" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yYgOiFNHZx-1", + "vscode": { + "languageId": "python" + } + }, + "outputs": [], "source": [ "# Classification val\n", "!bash data/scripts/get_imagenet.sh --val # download ImageNet val split (6.3G - 50000 images)\n", "!python classify/val.py --weights yolov5m-cls.pt --data ../datasets/imagenet --img 224 # validate" - ], - "metadata": { - "id": "yYgOiFNHZx-1" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aq4DPWGu0Bl1", + "vscode": { + "languageId": "python" + } + }, + "outputs": [], "source": [ "# Validate on COCO test. Zip results.json and submit to eval server at https://competitions.codalab.org/competitions/20794\n", "!bash data/scripts/get_coco.sh --test # download COCO test-dev2017 (7G - 40000 images, test 20000)\n", "!python val.py --weights yolov5x.pt --data coco.yaml --img 640 --iou 0.65 --half --task test" - ], - "metadata": { - "id": "aq4DPWGu0Bl1" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "VTRwsvA9u7ln" + "id": "VTRwsvA9u7ln", + "vscode": { + "languageId": "python" + } }, + "outputs": [], "source": [ "# TensorRT \n", "!pip install -U nvidia-tensorrt --index-url https://pypi.ngc.nvidia.com # install\n", "!python export.py --weights yolov5s.pt --include engine --imgsz 640 --device 0 # export\n", "!python detect.py --weights yolov5s.engine --imgsz 640 --device 0 # inference" - ], - "execution_count": null, - "outputs": [] + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "machine_shape": "hm", + "name": "YOLOv5 Tutorial", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "0856bea36ec148b68522ff9c9eb258d8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0ace3934ec6f4d36a1b3a9e086390926": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "35e03ce5090346c9ae602891470fc555": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76879f6f2aa54637a7a07faeea2bd684", + "max": 818322941, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_0ace3934ec6f4d36a1b3a9e086390926", + "value": 818322941 + } + }, + "574140e4c4bc48c9a171541a02cd0211": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_60b913d755b34d638478e30705a2dde1", + "placeholder": "​", + "style": "IPY_MODEL_0856bea36ec148b68522ff9c9eb258d8", + "value": "100%" + } + }, + "5966ba6e6f114d8c9d8d1d6b1bd4f4c7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "60b913d755b34d638478e30705a2dde1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "65881db1db8a4e9c930fab9172d45143": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "76879f6f2aa54637a7a07faeea2bd684": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9b8caa3522fc4cbab31e13b5dfc7808d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_574140e4c4bc48c9a171541a02cd0211", + "IPY_MODEL_35e03ce5090346c9ae602891470fc555", + "IPY_MODEL_c942c208e72d46568b476bb0f2d75496" + ], + "layout": "IPY_MODEL_65881db1db8a4e9c930fab9172d45143" + } + }, + "c942c208e72d46568b476bb0f2d75496": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6b7a2243e0c4beca714d99dceec23d6", + "placeholder": "​", + "style": "IPY_MODEL_5966ba6e6f114d8c9d8d1d6b1bd4f4c7", + "value": " 780M/780M [02:19<00:00, 6.24MB/s]" + } + }, + "d6b7a2243e0c4beca714d99dceec23d6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 949cb4478d2a7002a6e42246e38ca925aac09f90 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 11:12:26 +0000 Subject: [PATCH 10/41] add option to log class level metrics --- train.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/train.py b/train.py index 7c6811dc0338..daba29348883 100644 --- a/train.py +++ b/train.py @@ -505,6 +505,9 @@ def parse_opt(known=False): parser.add_argument("--comet_log_predictions", action="store_true", help="Comet: Log Predictions on Images from the Validation Set") + parser.add_argument("--comet_log_per_class_metrics", + action="store_true", + help="Comet: Log evaluation metrics for each class in the Validation Set") parser.add_argument("--comet_max_image_uploads", type=int, default=100, From aac31c29587bc0dc1facc25341053e7012771552 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 11:14:50 +0000 Subject: [PATCH 11/41] add support for class level metrics and confusion matrix --- utils/loggers/__init__.py | 5 ++- utils/loggers/comet/README.md | 54 ++++++++++++++------------ utils/loggers/comet/__init__.py | 69 ++++++++++++++++++++------------- val.py | 2 +- 4 files changed, 77 insertions(+), 53 deletions(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 2d8c3c8ab5aa..e675e4d6cd5d 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -197,7 +197,7 @@ def on_val_batch_end(self, batch_i, im, targets, paths, shapes, out): if self.comet_logger: self.comet_logger.on_val_batch_end(batch_i, im, targets, paths, shapes, out) - def on_val_end(self): + def on_val_end(self, nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix): # Callback runs on val end if self.wandb or self.clearml: files = sorted(self.save_dir.glob('val*.jpg')) @@ -206,6 +206,9 @@ def on_val_end(self): if self.clearml: self.clearml.log_debug_samples(files, title='Validation') + if self.comet_logger: + self.comet_logger.on_val_end(nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix) + def on_fit_epoch_end(self, vals, epoch, best_fitness, fi): # Callback runs at the end of each fit (train+val) epoch x = dict(zip(self.keys, vals)) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index 3263cfab1d9e..2050a6789551 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -90,44 +90,34 @@ Here is the full list of command line options for Comet ```shell --comet_mode COMET_MODE - Comet: Set whether to run Comet in online - or offline mode. + Comet: Set whether to run Comet in online or offline mode. --comet_save_model Comet: Set to save model checkpoints. --comet_model_name COMET_MODEL_NAME Comet: Set the name for the saved model. --comet_overwrite_checkpoints - Comet: Overwrite exsiting model - checkpoints. + Comet: Overwrite existing model checkpoints. --comet_checkpoint_filename [COMET_CHECKPOINT_FILENAME] - Comet: Name of the checkpoint file to save - to Comet.Set to 'all' to log all - checkpoints. - --comet_checkpoint_step COMET_CHECKPOINT_STEP + Comet: Name of the checkpoint file to save to Comet.Set to 'all' to + log all checkpoints. --comet_log_batch_metrics - Comet: Set to log batch level training - metrics. + Comet: Set to log batch level training metrics. --comet_log_batch_interval COMET_LOG_BATCH_INTERVAL - Comet: Logging frequency for batch level - training metrics. + Comet: Logging frequency for batch level training metrics. --comet_log_prediction_interval COMET_LOG_PREDICTION_INTERVAL - Comet: How often to log predictions.Applied - at batch level. + Comet: How often to log predictions.Applied at batch level. --comet_log_confusion_matrix - Comet: Log a Confusion Matrix for the - validation dataset. + Comet: Log a Confusion Matrix for the validation dataset. --comet_log_predictions - Comet: Log Predictions on Images from the - Validation Set + Comet: Log Predictions on Images from the Validation Set + --comet_log_per_class_metrics + Comet: Log evaluation metrics for each class in the Validation Set --comet_max_image_uploads COMET_MAX_IMAGE_UPLOADS - Comet: Maximum number of images to log to - Comet. + Comet: Maximum number of images to log to Comet. --comet_upload_dataset [COMET_UPLOAD_DATASET] - Comet: Upload Dataset to Comet as an - Artifact.Set to 'train', 'val' or 'test' to - upload a single dataset. + Comet: Upload Dataset to Comet as an Artifact.Set to 'train', 'val' + or 'test' to upload a single dataset. --comet_artifact COMET_ARTIFACT - Comet: Name of the Comet dataset Artifact - to download. + Comet: Name of the Comet dataset Artifact to download. ``` Let's take a look at these options. @@ -269,6 +259,20 @@ python train.py \ --comet_log_prediction_interval 2 ``` +### Logging Class Level Metrics + +Use the `comet_log_per_metrics` to log mAP, precision, recall, f1 for each class. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--comet_log_per_class_metrics +``` + ## Uploading a Dataset to Comet Artifacts If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5), you can do so using the `comet_upload_dataset` flag. diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index bfd840532570..4c7157e5fd54 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -29,13 +29,13 @@ # Model Saving Settings COMET_SAVE_MODEL = os.getenv("COMET_SAVE_MODEL", "false").lower() == "true" COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5") -COMET_OVERWRITE_CHECKPOINTS = (os.getenv("COMET_OVERWRITE_CHECKPOINTS", "false").lower() == "true") +COMET_OVERWRITE_CHECKPOINTS = os.getenv("COMET_OVERWRITE_CHECKPOINTS", "false").lower() == "true" # Dataset Artifact Settings COMET_UPLOAD_DATASET = os.getenv("COMET_UPLOAD_DATASET", "false").lower() == "true" # Evaluation Settings -COMET_LOG_CONFUSION_MATRIX = (os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true") +COMET_LOG_CONFUSION_MATRIX = os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true" COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "false").lower() == "true" COMET_MAX_IMAGE_UPLOADS = os.getenv("COMET_MAX_IMAGE_UPLOADS", 100) @@ -44,9 +44,10 @@ IOU_THRES = os.getenv("IOU_THRES", 0.6) # Batch Logging Settings -COMET_LOG_BATCH_METRICS = (os.getenv("COMET_LOG_BATCH_METRICS", "false").lower() == "true") +COMET_LOG_BATCH_METRICS = os.getenv("COMET_LOG_BATCH_METRICS", "false").lower() == "true" COMET_BATCH_LOGGING_INTERVAL = os.getenv("COMET_BATCH_LOGGING_INTERVAL", 1) COMET_PREDICTION_LOGGING_INTERVAL = os.getenv("COMET_PREDICTION_LOGGING_INTERVAL", 1) +COMET_LOG_PER_CLASS_METRICS = os.getenv("COMET_LOG_PER_CLASS_METRICS", "false").lower() == "true" RANK = int(os.getenv("RANK", -1)) @@ -128,15 +129,15 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.comet_log_confusion_matrix = (self.opt.comet_log_confusion_matrix if self.opt.comet_log_confusion_matrix else COMET_LOG_CONFUSION_MATRIX) - if self.comet_log_confusion_matrix: - if hasattr(self.opt, "conf_thres"): - self.conf_thres = self.opt.conf_thres - else: - self.conf_thres = CONF_THRES - if hasattr(self.opt, "iou_thres"): - self.iou_thres = self.opt.iou_thres - else: - self.iou_thres = IOU_THRES + + if hasattr(self.opt, "conf_thres"): + self.conf_thres = self.opt.conf_thres + else: + self.conf_thres = CONF_THRES + if hasattr(self.opt, "iou_thres"): + self.iou_thres = self.opt.iou_thres + else: + self.iou_thres = IOU_THRES self.comet_log_predictions = (self.opt.comet_log_predictions if self.opt.comet_log_predictions else COMET_LOG_PREDICTIONS) @@ -145,6 +146,8 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar if self.comet_log_predictions: self.metadata_dict = {} + self.comet_log_per_class_metrics = self.opt.comet_log_per_class_metrics if self.opt.comet_log_per_class_metrics else COMET_LOG_PER_CLASS_METRICS + # Check if running the Experiment with the Comet Optimizer if hasattr(self.opt, "comet_optimizer_id"): self.experiment.log_other("optimizer_id", self.opt.comet_optimizer_id) @@ -230,7 +233,6 @@ def log_model(self, path, opt, epoch, fitness_score, best_model=False): def log_predictions(self, image, labelsn, path, shape, predn): if self.logged_images_count >= self.max_images: return - detections = predn[predn[:, 4] > self.conf_thres] iou = box_iou(labelsn[:, 1:], detections[:, :4]) mask, _ = torch.where(iou > self.iou_thres) @@ -412,14 +414,15 @@ def on_train_end(self, files, save_dir, last, best, epoch, results): self.finish_run() def on_val_start(self): - self.confmat = ConfusionMatrix(nc=self.num_classes, conf=self.conf_thres, iou_thres=self.iou_thres) - return def on_val_batch_start(self): return def on_val_batch_end(self, batch_i, images, targets, paths, shapes, outputs): + if not self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0): + return + for si, pred in enumerate(outputs): if len(pred) == 0: continue @@ -428,34 +431,48 @@ def on_val_batch_end(self, batch_i, images, targets, paths, shapes, outputs): labels = targets[targets[:, 0] == si, 1:] shape = shapes[si] path = paths[si] - predn, labelsn = self.preprocess_prediction(image, labels, shape, pred) if labelsn is not None: - if self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0): - self.log_predictions(image, labelsn, path, shape, predn) - - if self.comet_log_confusion_matrix: - self.confmat.process_batch(predn, labelsn) + self.log_predictions(image, labelsn, path, shape, predn) return - def on_fit_epoch_end(self, result, epoch): - self.log_metrics(result, epoch=epoch) + def on_val_end(self, nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix): + if self.comet_log_per_class_metrics: + if self.num_classes > 1: + for i, c in enumerate(ap_class): + class_name = self.class_names[c] + self.experiment.log_metrics( + { + 'mAP@.5': ap50[i], + 'mAP@.5:.95': ap[i], + 'precision': p[i], + 'recall': r[i], + 'f1': f1[i], + 'true_positives': tp[i], + 'false_positives': fp[i], + 'support': nt[c]}, + prefix=class_name) if self.comet_log_confusion_matrix: + epoch = self.experiment.curr_epoch class_names = list(self.class_names.values()) - class_names.append("background-FN") - + class_names.append("background") num_classes = len(class_names) self.experiment.log_confusion_matrix( - matrix=self.confmat.matrix, + matrix=confusion_matrix.matrix, max_categories=num_classes, labels=class_names, epoch=epoch, + column_label='Actual Category', + row_label='Predicted Category', file_name=f"confusion-matrix-epoch-{epoch}.json", ) + def on_fit_epoch_end(self, result, epoch): + self.log_metrics(result, epoch=epoch) + def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): if ((epoch + 1) % self.opt.save_period == 0 and not final_epoch) and self.opt.save_period != -1: self.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi) diff --git a/val.py b/val.py index fdb4e5ec8853..3c2a8b722404 100644 --- a/val.py +++ b/val.py @@ -289,7 +289,7 @@ def run( # Plots if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) - callbacks.run('on_val_end') + callbacks.run('on_val_end', nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix) # Save JSON if save_json and len(jdict): From 1f0eb714b58d3974cfcb94e6662c37a2a9469091 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 12:13:57 +0000 Subject: [PATCH 12/41] handle errors when adding files to artifacts --- utils/loggers/comet/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 4c7157e5fd54..8f7a7cc59299 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -298,8 +298,14 @@ def add_assets_to_artifact(self, artifact, path, asset_path, split): for image_file, label_file in zip(img_paths, label_paths): image_logical_path, label_logical_path = map(lambda x: x.replace(f"{path}/", ""), [image_file, label_file]) - artifact.add(image_file, logical_path=image_logical_path, metadata={"split": split}) - artifact.add(label_file, logical_path=label_logical_path, metadata={"split": split}) + try: + artifact.add(image_file, logical_path=image_logical_path, metadata={"split": split}) + artifact.add(label_file, logical_path=label_logical_path, metadata={"split": split}) + except ValueError as e: + logger.error('COMET ERROR: Error adding file to Artifact. Skipping file.') + logger.error(f"COMET ERROR: {e}") + continue + return artifact def upload_dataset_artifact(self): From 24842bb7be03d872ec29375fcc3f8099f9823e46 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 12:38:25 +0000 Subject: [PATCH 13/41] fix typo --- utils/loggers/comet/comet_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py index f3dc0feeb629..1aab701ed44c 100644 --- a/utils/loggers/comet/comet_utils.py +++ b/utils/loggers/comet/comet_utils.py @@ -78,7 +78,7 @@ def set_opt_parameters(opt, experiment): setattr(opt, key, value) opt.resume = resume_string - # Save hyperparamers to YAML file + # Save hyperparameters to YAML file # Necessary to pass checks in training script save_dir = f"{opt.project}/{experiment.name}" os.makedirs(save_dir, exist_ok=True) From 7a87143d75c38b3c27a0400df18101d62c227044 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 13:08:35 +0000 Subject: [PATCH 14/41] clean resume workflow --- train.py | 2 +- utils/loggers/comet/comet_utils.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index daba29348883..43adf925baa3 100644 --- a/train.py +++ b/train.py @@ -533,7 +533,7 @@ def main(opt, callbacks=Callbacks()): check_comet_weights(opt) # Resume - if opt.resume and not check_wandb_resume(opt) or check_comet_resume( + if opt.resume and not check_wandb_resume(opt) and not check_comet_resume( opt) or opt.evolve: # resume from specified or most recent last.pt last = Path(check_file(opt.resume) if isinstance(opt.resume, str) else get_latest_run()) opt_yaml = last.parent.parent / 'opt.yaml' # train options yaml diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py index 1aab701ed44c..01fea00abb64 100644 --- a/utils/loggers/comet/comet_utils.py +++ b/utils/loggers/comet/comet_utils.py @@ -41,10 +41,11 @@ def download_model_checkpoint(opt, experiment): # Fetch latest checkpoint asset_id = model_asset_list[0]["assetId"] asset_filename = model_asset_list[0]["fileName"] - logger.info(f"COMET INFO: Checkpoint {checkpoint_filename} not found." + logger.info(f"COMET INFO: Checkpoint {checkpoint_filename} not found. " f"Defaulting to latest checkpoint {asset_filename}") else: + logger.info(f"COMET INFO: Downloading checkpoint {checkpoint_filename}") asset_filename = checkpoint_filename model_binary = experiment.get_asset(asset_id, return_type="binary", stream=False) @@ -69,6 +70,8 @@ def set_opt_parameters(opt, experiment): """ asset_list = experiment.get_asset_list() resume_string = opt.resume + override_checkpoint_filename = opt.comet_checkpoint_filename + for asset in asset_list: if asset["fileName"] == "opt.yaml": asset_id = asset["assetId"] @@ -88,6 +91,9 @@ def set_opt_parameters(opt, experiment): yaml.dump(opt.hyp, f) opt.hyp = hyp_yaml_path + if override_checkpoint_filename != opt.comet_checkpoint_filename: + opt.comet_checkpoint_filename = override_checkpoint_filename + def check_comet_weights(opt): """Downloads model weights from Comet and updates the From 501ba8a101915f29e5ae5e50afc760aed758fe02 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 13:55:59 +0000 Subject: [PATCH 15/41] updates for HPO --- utils/loggers/comet/hpo.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/utils/loggers/comet/hpo.py b/utils/loggers/comet/hpo.py index 01c2d1fffb96..6929fbb2452f 100644 --- a/utils/loggers/comet/hpo.py +++ b/utils/loggers/comet/hpo.py @@ -98,6 +98,9 @@ def get_args(known=False): parser.add_argument("--comet_log_predictions", action="store_true", help="Comet: Log Predictions on Images from the Validation Set") + parser.add_argument("--comet_log_per_class_metrics", + action="store_true", + help="Comet: Log evaluation metrics for each class in the Validation Set") parser.add_argument("--comet_max_image_uploads", type=int, default=100, @@ -109,6 +112,14 @@ def get_args(known=False): help=("Comet: Upload Dataset to Comet as an Artifact." "Set to 'train', 'val' or 'test' to upload a single dataset.")) parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") + parser.add_argument("--comet_optimizer_config", type=str, help="Comet: Path to a Comet Optimizer Config File.") + parser.add_argument("--comet_optimizer_id", type=str, help="Comet: ID of the Comet Optimizer sweep.") + parser.add_argument("--comet_optimizer_objective", type=str, help="Comet: Set to 'minimize' or 'maximize'.") + parser.add_argument("--comet_optimizer_metric", type=str, help="Comet: Metric to Optimize.") + parser.add_argument("--comet_optimizer_workers", + type=int, + default=1, + help="Comet: Number of Parallel Workers to use with the Comet Optimizer.") return parser.parse_known_args()[0] if known else parser.parse_args() From 9ae182ebf08fb13f19649b3f8bbb642f7fa779ba Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 14:23:30 +0000 Subject: [PATCH 16/41] update comet README --- utils/loggers/comet/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index 2050a6789551..dd73be824fe6 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -2,14 +2,13 @@ # YOLOv5 with Comet -This guide will cover how to use YOLOv5 with [Comet](https://www.comet.com/site/?ref=yolov5) +This guide will cover how to use YOLOv5 with [Comet](https://www.comet.com/site/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) # About Comet Comet builds tools that help data scientists, engineers, and team leaders accelerate and optimize machine learning and deep learning models. -Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1?ref=yolov5)! - +Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration)! Comet makes sure you never lose track of your work and makes it easy to share results and collaborate across teams of all sizes! # Getting Started @@ -55,7 +54,7 @@ That's it! Comet will automatically log your hyperparameters, command line argum yolo-ui # Try out an Example! -Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/353f9734261348b59b883660bcd62256?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5) +Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/353f9734261348b59b883660bcd62256?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) Or better yet, try it out yourself in this Colab Notebook @@ -215,7 +214,7 @@ python train.py \ You can log model predictions and the associated images using `comet_log_predictions`. Predictions can be visualized using Comet's Object Detection Custom Panel -Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1?ref=yolov5) +Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) ```shell python train.py \ @@ -275,7 +274,7 @@ python train.py \ ## Uploading a Dataset to Comet Artifacts -If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5), you can do so using the `comet_upload_dataset` flag. +If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration), you can do so using the `comet_upload_dataset` flag. The dataset be organized in the way described in the [YOLOv5 documentation](https://docs.ultralytics.com/tutorials/train-custom-datasets/#3-organize-directories). The dataset config `yaml` file must follow the same format as that of the `coco128.yaml` file. @@ -337,7 +336,7 @@ To configure the Comet Optimizer, you will have to create a JSON file with the i ```shell python utils/loggers/comet/hpo.py \ - --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" \ + --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" ``` The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional Comet related arguments to your sweep simply add them after @@ -367,6 +366,6 @@ comet optimizer -j utils/loggers/comet/hpo.py \ ### Visualizing Results -Comet provides a number of ways to visualize the results of your sweep. Take a look at a [project with a completed sweep here](https://www.comet.com/examples/comet-example-yolov5/view/PrlArHGuuhDTKC1UuBmTtOSXD/panels?ref=yolov5) +Comet provides a number of ways to visualize the results of your sweep. Take a look at a [project with a completed sweep here](https://www.comet.com/examples/comet-example-yolov5/view/PrlArHGuuhDTKC1UuBmTtOSXD/panels?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) hyperparameter-yolo \ No newline at end of file From 177ac97b416103aaf0ead24629bd95446caa9fd6 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 14:25:09 +0000 Subject: [PATCH 17/41] fix typo in comet README --- utils/loggers/comet/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index dd73be824fe6..d58fe3b76cad 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -260,7 +260,7 @@ python train.py \ ### Logging Class Level Metrics -Use the `comet_log_per_metrics` to log mAP, precision, recall, f1 for each class. +Use the `comet_log_per_class_metrics` to log mAP, precision, recall, f1 for each class. ```shell python train.py \ From e1c09d80fb7170a1a50209c81f6b91310938d379 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 15:15:05 +0000 Subject: [PATCH 18/41] update code snippets in comet README --- utils/loggers/comet/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index d58fe3b76cad..ce890ef1086b 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -194,7 +194,7 @@ python train.py \ --batch 16 \ --epochs 5 \ --data coco128.yaml \ ---weights "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" \ +--weights "comet://" \ ``` By default, Comet will download the most recent checkpoint, this can be configured by specifying the checkpoint filename. You can specify a checkpoint for a particular epoch @@ -206,7 +206,7 @@ python train.py \ --batch 16 \ --epochs 5 \ --data coco128.yaml \ ---weights "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" \ +--weights "comet://" \ --comet_checkpoint_filename "last_epoch_2.pt" ``` @@ -323,7 +323,7 @@ This will restore the run to its state before the interruption, which includes r ```shell python train.py \ ---resume "comet://examples/comet-example-yolov5/353f9734261348b59b883660bcd62256" +--resume "comet://" ``` ## Hyperparameter Search with the Comet Optimizer From eb12ea5cb59f959892f6d307b289e3c9e3a0c634 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 15:15:44 +0000 Subject: [PATCH 19/41] update comet links in tutorial --- tutorial.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index 9909c0c3b2fa..74390672380d 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -521,7 +521,7 @@ "metadata": {}, "source": [ "## Comet Logging and Visualization 🌟 NEW\n", - "[Comet](https://www.comet.com/site/?ref=yolov5) is integrated with YOLOv5. Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/docs/v2/guides/comet-dashboard/code-panels/about-panels/?ref=yolov5)!\n", + "[Comet](https://www.comet.com/site/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) is integrated with YOLOv5. Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/docs/v2/guides/comet-dashboard/code-panels/about-panels/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration)!\n", "\n", "Comet makes sure you never lose track of your work and makes it easy to share results and collaborate across teams of all sizes! \n", "\n", From d2f931f988d16648e068fc31fe994c1f4808a51a Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 30 Aug 2022 16:40:43 +0000 Subject: [PATCH 20/41] updated links --- utils/loggers/comet/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index ce890ef1086b..b4b4beecff02 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -54,7 +54,7 @@ That's it! Comet will automatically log your hyperparameters, command line argum yolo-ui # Try out an Example! -Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/353f9734261348b59b883660bcd62256?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) +Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/ca87319ab1684f659a0ea6c625ef33f5?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) Or better yet, try it out yourself in this Colab Notebook @@ -332,7 +332,7 @@ YOLOv5 is also integrated with Comet's Optimizer, making is simple to visualie h ### Configuring an Optimizer Sweep -To configure the Comet Optimizer, you will have to create a JSON file with the information about the sweep. An example file has been provided [here](optimizer_config.json) +To configure the Comet Optimizer, you will have to create a JSON file with the information about the sweep. An example file has been provided in `utils/loggers/comet/optimizer_config.json` ```shell python utils/loggers/comet/hpo.py \ From 5f959dfcdbbee09fc41253d78b3a2e58005c5a1c Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 31 Aug 2022 06:29:53 +0000 Subject: [PATCH 21/41] change optimizer batch size param and update comet README image --- utils/loggers/comet/README.md | 2 +- utils/loggers/comet/optimizer_config.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index b4b4beecff02..37619be2a858 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -51,7 +51,7 @@ python train.py --img 640 --batch 16 --epochs 5 --data coco128.yaml --weights yo That's it! Comet will automatically log your hyperparameters, command line arguments, training and valiation metrics. You can visualize and analyze your runs in the Comet UI -yolo-ui +yolo-ui # Try out an Example! Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/ca87319ab1684f659a0ea6c625ef33f5?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) diff --git a/utils/loggers/comet/optimizer_config.json b/utils/loggers/comet/optimizer_config.json index d0b3857ff96f..83ddddab6f20 100644 --- a/utils/loggers/comet/optimizer_config.json +++ b/utils/loggers/comet/optimizer_config.json @@ -13,8 +13,7 @@ "values": [ 16, 32, - 64, - 128 + 64 ] }, "box": { From ee7b82c6b807adf8a62f31b414ba15932d999f1a Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 31 Aug 2022 06:40:53 +0000 Subject: [PATCH 22/41] update comet section in tutorial --- tutorial.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index 74390672380d..d46529f27108 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -543,12 +543,12 @@ "python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt\n", "```\n", "\n", - "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/comet). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?ref=yolov5) \n", + "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/comet). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) \n", "\n", "Get started by trying out the Comet Colab Notebook:\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing)\n", "\n", - "\"yolo-ui\"" + "\"yolo-ui\"" ] }, { From 304e270fc57f18d58d12566f8f3f8bedb6e805a8 Mon Sep 17 00:00:00 2001 From: DN6 Date: Fri, 2 Sep 2022 09:14:26 +0000 Subject: [PATCH 23/41] use prexisting cmd line flags to configure logger --- train.py | 3 +- utils/loggers/comet/__init__.py | 91 ++++++++++++++++-------------- utils/loggers/comet/comet_utils.py | 2 +- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/train.py b/train.py index 43adf925baa3..925feb6da7d3 100644 --- a/train.py +++ b/train.py @@ -100,6 +100,7 @@ def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictio weights, epochs, hyp, batch_size = opt.weights, opt.epochs, opt.hyp, opt.batch_size if loggers.comet_logger: data_dict = loggers.comet_logger.data_dict + weights, epochs, hyp, batch_size = opt.weights, opt.epochs, opt.hyp, opt.batch_size # Register actions for k in methods(loggers): callbacks.register_action(k, callback=getattr(loggers, k)) @@ -530,8 +531,6 @@ def main(opt, callbacks=Callbacks()): check_git_status() check_requirements() - check_comet_weights(opt) - # Resume if opt.resume and not check_wandb_resume(opt) and not check_comet_resume( opt) or opt.evolve: # resume from specified or most recent last.pt diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 8f7a7cc59299..e046068cd860 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -24,6 +24,8 @@ from utils.general import check_dataset, scale_coords, xywh2xyxy from utils.metrics import ConfusionMatrix, box_iou +from .comet_utils import COMET_PREFIX, check_comet_resume, check_comet_weights + COMET_MODE = os.getenv("COMET_MODE", "online") # Model Saving Settings @@ -65,21 +67,18 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.hyp = hyp # Comet Flags - self.comet_mode = self.opt.comet_mode if self.opt.comet_mode else COMET_MODE + self.comet_mode = COMET_MODE - self.save_model = (opt.comet_save_model if opt.comet_save_model else COMET_SAVE_MODEL) - self.model_name = (opt.comet_model_name if opt.comet_model_name else COMET_MODEL_NAME) - self.overwrite_checkpoints = (opt.comet_overwrite_checkpoints - if opt.comet_overwrite_checkpoints else COMET_OVERWRITE_CHECKPOINTS) + self.save_model = opt.save_period > -1 + self.model_name = COMET_MODEL_NAME + self.overwrite_checkpoints = COMET_OVERWRITE_CHECKPOINTS # Batch Logging Settings - self.log_batch_metrics = (opt.comet_log_batch_metrics - if opt.comet_log_batch_metrics else COMET_LOG_BATCH_METRICS) - self.comet_log_batch_interval = (opt.comet_log_batch_interval - if opt.comet_log_batch_interval else COMET_BATCH_LOGGING_INTERVAL) + self.log_batch_metrics = COMET_LOG_BATCH_METRICS + self.comet_log_batch_interval = COMET_BATCH_LOGGING_INTERVAL # Dataset Artifact Settings - self.upload_dataset = (self.opt.comet_upload_dataset if self.opt.comet_upload_dataset else COMET_UPLOAD_DATASET) + self.upload_dataset = self.opt.upload_dataset if self.opt.upload_dataset else COMET_UPLOAD_DATASET self.resume = self.opt.resume # Default parameters to pass to Experiment objects @@ -101,34 +100,31 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.num_classes = self.data_dict["nc"] self.logged_images_count = 0 - self.max_images = (self.opt.comet_max_image_uploads - if self.opt.comet_max_image_uploads else COMET_MAX_IMAGE_UPLOADS) - if self.experiment is not None: - if run_id is None: - self.experiment.log_other("Created from", "YOLOv5") - - if not isinstance(self.experiment, comet_ml.OfflineExperiment): - workspace, project_name, experiment_id = self.experiment.url.split("/")[-3:] - self.experiment.log_other( - "Run Path", - f"{workspace}/{project_name}/{experiment_id}", - ) - self.log_parameters(vars(opt)) - self.log_parameters(self.opt.hyp) - self.log_asset_data( - self.opt.hyp, - name="hyperparameters.json", - metadata={"type": "hyp-config-file"}, - ) - self.log_asset( - f"{self.opt.save_dir}/opt.yaml", - metadata={"type": "opt-config-file"}, + self.max_images = COMET_MAX_IMAGE_UPLOADS + + if run_id is None: + self.experiment.log_other("Created from", "YOLOv5") + if not isinstance(self.experiment, comet_ml.OfflineExperiment): + workspace, project_name, experiment_id = self.experiment.url.split("/")[-3:] + self.experiment.log_other( + "Run Path", + f"{workspace}/{project_name}/{experiment_id}", ) - if not self.opt.comet_artifact: - self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) + self.log_parameters(vars(opt)) + self.log_parameters(self.opt.hyp) + self.log_asset_data( + self.opt.hyp, + name="hyperparameters.json", + metadata={"type": "hyp-config-file"}, + ) + self.log_asset( + f"{self.opt.save_dir}/opt.yaml", + metadata={"type": "opt-config-file"}, + ) + if not self.opt.comet_artifact: + self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) - self.comet_log_confusion_matrix = (self.opt.comet_log_confusion_matrix - if self.opt.comet_log_confusion_matrix else COMET_LOG_CONFUSION_MATRIX) + self.comet_log_confusion_matrix = COMET_LOG_CONFUSION_MATRIX if hasattr(self.opt, "conf_thres"): self.conf_thres = self.opt.conf_thres @@ -139,14 +135,12 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar else: self.iou_thres = IOU_THRES - self.comet_log_predictions = (self.opt.comet_log_predictions - if self.opt.comet_log_predictions else COMET_LOG_PREDICTIONS) - self.comet_log_prediction_interval = (opt.comet_log_prediction_interval if opt.comet_log_prediction_interval - else COMET_PREDICTION_LOGGING_INTERVAL) + self.comet_log_predictions = self.opt.bbox_interval > -1 + self.comet_log_prediction_interval = self.opt.bbox_interval if self.comet_log_predictions: self.metadata_dict = {} - self.comet_log_per_class_metrics = self.opt.comet_log_per_class_metrics if self.opt.comet_log_per_class_metrics else COMET_LOG_PER_CLASS_METRICS + self.comet_log_per_class_metrics = COMET_LOG_PER_CLASS_METRICS # Check if running the Experiment with the Comet Optimizer if hasattr(self.opt, "comet_optimizer_id"): @@ -230,6 +224,18 @@ def log_model(self, path, opt, epoch, fitness_score, best_model=False): overwrite=self.overwrite_checkpoints, ) + def check_dataset(self, data_file): + with open(data_file) as f: + data_config = yaml.safe_load(f) + + if data_config['path'].startswith(COMET_PREFIX): + path = data_config['path'].replace(COMET_PREFIX, "") + data_dict = self.download_dataset_artifact(path) + + return data_dict + + return check_dataset(data_file) + def log_predictions(self, image, labelsn, path, shape, predn): if self.logged_images_count >= self.max_images: return @@ -487,5 +493,4 @@ def on_params_update(self, params): self.log_parameters(params) def finish_run(self): - if self.experiment is not None: - self.experiment.end() + self.experiment.end() diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py index 01fea00abb64..78130b9b7b53 100644 --- a/utils/loggers/comet/comet_utils.py +++ b/utils/loggers/comet/comet_utils.py @@ -32,7 +32,7 @@ def download_model_checkpoint(opt, experiment): reverse=True, ) - checkpoint_filename = opt.comet_checkpoint_filename + checkpoint_filename = urlparse(opt.weights) logged_checkpoint_map = {asset["fileName"]: asset["assetId"] for asset in model_asset_list} asset_id = logged_checkpoint_map.get(checkpoint_filename) From b784a822010174b6b1a147ddd56865d975f70e2a Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 04:52:24 +0000 Subject: [PATCH 24/41] update artifact upload/download flow --- utils/loggers/comet/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index e046068cd860..0525da6dece6 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -2,10 +2,16 @@ import json import logging import os +import sys from pathlib import Path logger = logging.getLogger(__name__) +FILE = Path(__file__).resolve() +ROOT = FILE.parents[3] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + try: import comet_ml @@ -22,7 +28,7 @@ from utils.dataloaders import img2label_paths from utils.general import check_dataset, scale_coords, xywh2xyxy -from utils.metrics import ConfusionMatrix, box_iou +from utils.metrics import box_iou from .comet_utils import COMET_PREFIX, check_comet_resume, check_comet_weights @@ -90,12 +96,7 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.default_experiment_kwargs.update(experiment_kwargs) self.experiment = self._get_experiment(self.comet_mode, run_id) - if self.opt.comet_artifact: - self.data_dict = self.download_dataset_artifact(self.opt.comet_artifact) - - else: - self.data_dict = check_dataset(self.opt.data) - + self.data_dict = self.check_dataset(self.opt.data) self.class_names = self.data_dict["names"] self.num_classes = self.data_dict["nc"] @@ -303,7 +304,8 @@ def add_assets_to_artifact(self, artifact, path, asset_path, split): label_paths = img2label_paths(img_paths) for image_file, label_file in zip(img_paths, label_paths): - image_logical_path, label_logical_path = map(lambda x: x.replace(f"{path}/", ""), [image_file, label_file]) + image_logical_path, label_logical_path = map(lambda x: os.path.relpath(x, path), [image_file, label_file]) + try: artifact.add(image_file, logical_path=image_logical_path, metadata={"split": split}) artifact.add(label_file, logical_path=label_logical_path, metadata={"split": split}) @@ -316,7 +318,7 @@ def add_assets_to_artifact(self, artifact, path, asset_path, split): def upload_dataset_artifact(self): dataset_name = self.data_dict.get("dataset_name", "yolov5-dataset") - path = self.data_dict["path"] + path = str((ROOT / Path(self.data_dict["path"])).resolve()) metadata = self.data_dict.copy() for key in ["train", "val", "test"]: @@ -340,11 +342,12 @@ def upload_dataset_artifact(self): def download_dataset_artifact(self, artifact_path): logged_artifact = self.experiment.get_artifact(artifact_path) - logged_artifact.download(self.opt.save_dir) + artifact_save_dir = str(Path(self.opt.save_dir) / logged_artifact.name) + logged_artifact.download(artifact_save_dir) metadata = logged_artifact.metadata data_dict = metadata.copy() - data_dict["path"] = self.opt.save_dir + data_dict["path"] = artifact_save_dir data_dict["names"] = {int(k): v for k, v in metadata.get("names").items()} data_dict = self.update_data_paths(data_dict) From 443eb068b23f905fadc93be63febe8b6861e2a1a Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 05:06:28 +0000 Subject: [PATCH 25/41] remove come remove comet logger specific cmd line args --- train.py | 48 +----------------------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/train.py b/train.py index 925feb6da7d3..addf49b6acb1 100644 --- a/train.py +++ b/train.py @@ -52,7 +52,7 @@ init_seeds, intersect_dicts, labels_to_class_weights, labels_to_image_weights, methods, one_cycle, print_args, print_mutation, strip_optimizer, yaml_save) from utils.loggers import Loggers -from utils.loggers.comet.comet_utils import check_comet_resume, check_comet_weights +from utils.loggers.comet.comet_utils import check_comet_resume from utils.loggers.wandb.wandb_utils import check_wandb_resume from utils.loss import ComputeLoss from utils.metrics import fitness @@ -475,52 +475,6 @@ def parse_opt(known=False): parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval') parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') - # Comet Arguments - parser.add_argument("--comet_mode", type=str, help="Comet: Set whether to run Comet in online or offline mode.") - parser.add_argument("--comet_save_model", action="store_true", help="Comet: Set to save model checkpoints.") - parser.add_argument("--comet_model_name", type=str, help="Comet: Set the name for the saved model.") - parser.add_argument("--comet_overwrite_checkpoints", - action="store_true", - help="Comet: Overwrite existing model checkpoints.") - parser.add_argument("--comet_checkpoint_filename", - nargs="?", - type=str, - default="best.pt", - help=("Comet: Name of the checkpoint file to save to Comet." - "Set to 'all' to log all checkpoints.")) - parser.add_argument("--comet_log_batch_metrics", - action="store_true", - help="Comet: Set to log batch level training metrics.") - parser.add_argument("--comet_log_batch_interval", - type=int, - default=1, - help="Comet: Logging frequency for batch level training metrics.") - parser.add_argument("--comet_log_prediction_interval", - type=int, - default=1, - help=("Comet: How often to log predictions." - "Applied at batch level.")) - parser.add_argument("--comet_log_confusion_matrix", - action="store_true", - help="Comet: Log a Confusion Matrix for the validation dataset.") - parser.add_argument("--comet_log_predictions", - action="store_true", - help="Comet: Log Predictions on Images from the Validation Set") - parser.add_argument("--comet_log_per_class_metrics", - action="store_true", - help="Comet: Log evaluation metrics for each class in the Validation Set") - parser.add_argument("--comet_max_image_uploads", - type=int, - default=100, - help="Comet: Maximum number of images to log to Comet.") - parser.add_argument("--comet_upload_dataset", - nargs="?", - const=True, - default=False, - help=("Comet: Upload Dataset to Comet as an Artifact." - "Set to 'train', 'val' or 'test' to upload a single dataset.")) - parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") - return parser.parse_known_args()[0] if known else parser.parse_args() From bb595d8ac72cc969aa81ea4f39c20dced3662416 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 05:07:18 +0000 Subject: [PATCH 26/41] move downloading weights into comet logger code --- utils/loggers/comet/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 0525da6dece6..4836ec8382ad 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -72,6 +72,8 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.opt = opt self.hyp = hyp + check_comet_weights(self.opt) + # Comet Flags self.comet_mode = COMET_MODE From 44a8ba03f70f1841f4814c0db24f53a2a530b458 Mon Sep 17 00:00:00 2001 From: Ayush Chaurasia Date: Mon, 5 Sep 2022 11:57:04 +0530 Subject: [PATCH 27/41] remove extra argparse --- train.py | 57 ++++----------------------------- utils/loggers/comet/__init__.py | 8 ++--- 2 files changed, 8 insertions(+), 57 deletions(-) diff --git a/train.py b/train.py index 36d73ae2bf5e..6516ac193f1f 100644 --- a/train.py +++ b/train.py @@ -466,57 +466,12 @@ def parse_opt(known=False): parser.add_argument('--seed', type=int, default=0, help='Global training seed') parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify') - # Weights & Biases arguments - parser.add_argument('--entity', default=None, help='W&B: Entity') - parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='W&B: Upload data, "val" option') - parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval') - parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') - - # Comet Arguments - parser.add_argument("--comet_mode", type=str, help="Comet: Set whether to run Comet in online or offline mode.") - parser.add_argument("--comet_save_model", action="store_true", help="Comet: Set to save model checkpoints.") - parser.add_argument("--comet_model_name", type=str, help="Comet: Set the name for the saved model.") - parser.add_argument("--comet_overwrite_checkpoints", - action="store_true", - help="Comet: Overwrite existing model checkpoints.") - parser.add_argument("--comet_checkpoint_filename", - nargs="?", - type=str, - default="best.pt", - help=("Comet: Name of the checkpoint file to save to Comet." - "Set to 'all' to log all checkpoints.")) - parser.add_argument("--comet_log_batch_metrics", - action="store_true", - help="Comet: Set to log batch level training metrics.") - parser.add_argument("--comet_log_batch_interval", - type=int, - default=1, - help="Comet: Logging frequency for batch level training metrics.") - parser.add_argument("--comet_log_prediction_interval", - type=int, - default=1, - help=("Comet: How often to log predictions." - "Applied at batch level.")) - parser.add_argument("--comet_log_confusion_matrix", - action="store_true", - help="Comet: Log a Confusion Matrix for the validation dataset.") - parser.add_argument("--comet_log_predictions", - action="store_true", - help="Comet: Log Predictions on Images from the Validation Set") - parser.add_argument("--comet_log_per_class_metrics", - action="store_true", - help="Comet: Log evaluation metrics for each class in the Validation Set") - parser.add_argument("--comet_max_image_uploads", - type=int, - default=100, - help="Comet: Maximum number of images to log to Comet.") - parser.add_argument("--comet_upload_dataset", - nargs="?", - const=True, - default=False, - help=("Comet: Upload Dataset to Comet as an Artifact." - "Set to 'train', 'val' or 'test' to upload a single dataset.")) - parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") + # Logger arguments + parser.add_argument('--entity', default=None, help='Entity') + parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='Upload data, "val" option') + parser.add_argument('--bbox_interval', type=int, default=-1, help='Set bounding-box image logging interval') + parser.add_argument('--artifact_alias', type=str, default='latest', help='Version of dataset artifact to use') + return parser.parse_known_args()[0] if known else parser.parse_args() diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index e046068cd860..862dba64321d 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -90,11 +90,7 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.default_experiment_kwargs.update(experiment_kwargs) self.experiment = self._get_experiment(self.comet_mode, run_id) - if self.opt.comet_artifact: - self.data_dict = self.download_dataset_artifact(self.opt.comet_artifact) - - else: - self.data_dict = check_dataset(self.opt.data) + self.data_dict = check_dataset(self.opt.data) self.class_names = self.data_dict["names"] self.num_classes = self.data_dict["nc"] @@ -121,7 +117,7 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar f"{self.opt.save_dir}/opt.yaml", metadata={"type": "opt-config-file"}, ) - if not self.opt.comet_artifact: + if not self.opt.upload_dataset: self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) self.comet_log_confusion_matrix = COMET_LOG_CONFUSION_MATRIX From 4a0f4223cc6543cf24441dabb484b975df2ab1f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 06:27:54 +0000 Subject: [PATCH 28/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/train.py b/train.py index 6516ac193f1f..6906a091605c 100644 --- a/train.py +++ b/train.py @@ -471,7 +471,6 @@ def parse_opt(known=False): parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='Upload data, "val" option') parser.add_argument('--bbox_interval', type=int, default=-1, help='Set bounding-box image logging interval') parser.add_argument('--artifact_alias', type=str, default='latest', help='Version of dataset artifact to use') - return parser.parse_known_args()[0] if known else parser.parse_args() From f15216ce2931d3beeb6a99eb36d4216927b98f8c Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 06:40:54 +0000 Subject: [PATCH 29/41] change checkpoint logging flow to follow offline logger --- utils/loggers/comet/__init__.py | 37 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 4836ec8382ad..16a332826d08 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -35,9 +35,7 @@ COMET_MODE = os.getenv("COMET_MODE", "online") # Model Saving Settings -COMET_SAVE_MODEL = os.getenv("COMET_SAVE_MODEL", "false").lower() == "true" COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5") -COMET_OVERWRITE_CHECKPOINTS = os.getenv("COMET_OVERWRITE_CHECKPOINTS", "false").lower() == "true" # Dataset Artifact Settings COMET_UPLOAD_DATASET = os.getenv("COMET_UPLOAD_DATASET", "false").lower() == "true" @@ -79,7 +77,6 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.save_model = opt.save_period > -1 self.model_name = COMET_MODEL_NAME - self.overwrite_checkpoints = COMET_OVERWRITE_CHECKPOINTS # Batch Logging Settings self.log_batch_metrics = COMET_LOG_BATCH_METRICS @@ -124,8 +121,6 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar f"{self.opt.save_dir}/opt.yaml", metadata={"type": "opt-config-file"}, ) - if not self.opt.comet_artifact: - self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) self.comet_log_confusion_matrix = COMET_LOG_CONFUSION_MATRIX @@ -145,6 +140,14 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.comet_log_per_class_metrics = COMET_LOG_PER_CLASS_METRICS + self.experiment.log_others({ + "comet_mode": COMET_MODE, + "comet_max_image_uploads": COMET_MAX_IMAGE_UPLOADS, + "comet_log_per_class_metrics": COMET_LOG_PER_CLASS_METRICS, + "comet_log_batch_metrics": COMET_LOG_BATCH_METRICS, + "comet_log_confusion_matrix": COMET_LOG_CONFUSION_MATRIX, + "comet_model_name": COMET_MODEL_NAME,}) + # Check if running the Experiment with the Comet Optimizer if hasattr(self.opt, "comet_optimizer_id"): self.experiment.log_other("optimizer_id", self.opt.comet_optimizer_id) @@ -206,25 +209,16 @@ def log_model(self, path, opt, epoch, fitness_score, best_model=False): "save_period": opt.save_period, "total_epochs": opt.epochs,} - if opt.comet_checkpoint_filename == "all": - model_path = str(path) - model_files = glob.glob(f"{path}/*.pt") - - else: - model_files = [str(path) + f"/{opt.comet_checkpoint_filename}"] - + model_files = glob.glob(f"{path}/*.pt") for model_path in model_files: - if not self.overwrite_checkpoints: - name = f"{Path(model_path).stem}_epoch_{epoch}.pt" - else: - name = Path(model_path).name + name = Path(model_path).name self.experiment.log_model( self.model_name, file_or_folder=model_path, file_name=name, metadata=model_metadata, - overwrite=self.overwrite_checkpoints, + overwrite=True, ) def check_dataset(self, data_file): @@ -237,6 +231,8 @@ def check_dataset(self, data_file): return data_dict + self.log_asset(self.opt.data, metadata={"type": "data-config-file"}) + return check_dataset(data_file) def log_predictions(self, image, labelsn, path, shape, predn): @@ -411,16 +407,13 @@ def on_train_end(self, files, save_dir, last, best, epoch, results): if not self.opt.evolve: model_path = str(best if best.exists() else last) - if not self.overwrite_checkpoints: - name = name = f"{Path(model_path).stem}_epoch_{epoch}.pt" - else: - name = Path(model_path).name + name = Path(model_path).name if self.save_model: self.experiment.log_model( self.model_name, file_or_folder=model_path, file_name=name, - overwrite=self.overwrite_checkpoints, + overwrite=True, ) # Check if running Experiment with Comet Optimizer From b08a43cc1711e0c78299eb2b303e8536aff10cde Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 08:51:26 +0000 Subject: [PATCH 30/41] update resume flow --- utils/loggers/comet/__init__.py | 4 +--- utils/loggers/comet/comet_utils.py | 35 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 16a332826d08..979cdb8bec31 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -30,7 +30,7 @@ from utils.general import check_dataset, scale_coords, xywh2xyxy from utils.metrics import box_iou -from .comet_utils import COMET_PREFIX, check_comet_resume, check_comet_weights +COMET_PREFIX = "comet://" COMET_MODE = os.getenv("COMET_MODE", "online") @@ -70,8 +70,6 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar self.opt = opt self.hyp = hyp - check_comet_weights(self.opt) - # Comet Flags self.comet_mode = COMET_MODE diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py index 78130b9b7b53..3cbd45156b57 100644 --- a/utils/loggers/comet/comet_utils.py +++ b/utils/loggers/comet/comet_utils.py @@ -13,13 +13,14 @@ COMET_PREFIX = "comet://" COMET_MODEL_NAME = os.getenv("COMET_MODEL_NAME", "yolov5") +COMET_DEFAULT_CHECKPOINT_FILENAME = os.getenv("COMET_DEFAULT_CHECKPOINT_FILENAME", "last.pt") def download_model_checkpoint(opt, experiment): model_dir = f"{opt.project}/{experiment.name}" os.makedirs(model_dir, exist_ok=True) - model_name = opt.comet_model_name if opt.comet_model_name else COMET_MODEL_NAME + model_name = COMET_MODEL_NAME model_asset_list = experiment.get_model_asset_list(model_name) if len(model_asset_list) == 0: @@ -31,22 +32,24 @@ def download_model_checkpoint(opt, experiment): key=lambda x: x["step"], reverse=True, ) - - checkpoint_filename = urlparse(opt.weights) logged_checkpoint_map = {asset["fileName"]: asset["assetId"] for asset in model_asset_list} - asset_id = logged_checkpoint_map.get(checkpoint_filename) - try: - if asset_id is None: - # Fetch latest checkpoint - asset_id = model_asset_list[0]["assetId"] - asset_filename = model_asset_list[0]["fileName"] - logger.info(f"COMET INFO: Checkpoint {checkpoint_filename} not found. " - f"Defaulting to latest checkpoint {asset_filename}") + resource_url = urlparse(opt.weights) + checkpoint_filename = resource_url.query - else: - logger.info(f"COMET INFO: Downloading checkpoint {checkpoint_filename}") - asset_filename = checkpoint_filename + if checkpoint_filename: + asset_id = logged_checkpoint_map.get(checkpoint_filename) + else: + asset_id = logged_checkpoint_map.get(COMET_DEFAULT_CHECKPOINT_FILENAME) + checkpoint_filename = COMET_DEFAULT_CHECKPOINT_FILENAME + + if asset_id is None: + logger.error(f"COMET ERROR: Checkpoint {checkpoint_filename} not found in the given Experiment") + return + + try: + logger.info(f"COMET INFO: Downloading checkpoint {checkpoint_filename}") + asset_filename = checkpoint_filename model_binary = experiment.get_asset(asset_id, return_type="binary", stream=False) model_download_path = f"{model_dir}/{asset_filename}" @@ -70,7 +73,6 @@ def set_opt_parameters(opt, experiment): """ asset_list = experiment.get_asset_list() resume_string = opt.resume - override_checkpoint_filename = opt.comet_checkpoint_filename for asset in asset_list: if asset["fileName"] == "opt.yaml": @@ -91,9 +93,6 @@ def set_opt_parameters(opt, experiment): yaml.dump(opt.hyp, f) opt.hyp = hyp_yaml_path - if override_checkpoint_filename != opt.comet_checkpoint_filename: - opt.comet_checkpoint_filename = override_checkpoint_filename - def check_comet_weights(opt): """Downloads model weights from Comet and updates the From 72169b61824381dd44185dab9c308d31a7f5c52f Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 08:52:14 +0000 Subject: [PATCH 31/41] add comet logger to remote dataset property --- utils/loggers/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index d7d482c6fe60..f29debb76907 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -142,6 +142,8 @@ def remote_dataset(self): data_dict = self.clearml.data_dict if self.wandb: data_dict = self.wandb.data_dict + if self.comet_logger: + data_dict = self.comet_logger.data_dict return data_dict From 075e48aa5579432b39e653d60d2e473f9880a759 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 08:56:48 +0000 Subject: [PATCH 32/41] update cmd line args in hpo --- utils/loggers/comet/hpo.py | 44 -------------------------------------- 1 file changed, 44 deletions(-) diff --git a/utils/loggers/comet/hpo.py b/utils/loggers/comet/hpo.py index 6929fbb2452f..eab4df9978cf 100644 --- a/utils/loggers/comet/hpo.py +++ b/utils/loggers/comet/hpo.py @@ -68,50 +68,6 @@ def get_args(known=False): parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') # Comet Arguments - parser.add_argument("--comet_mode", type=str, help="Comet: Set whether to run Comet in online or offline mode.") - parser.add_argument("--comet_save_model", action="store_true", help="Comet: Set to save model checkpoints.") - parser.add_argument("--comet_model_name", type=str, help="Comet: Set the name for the saved model.") - parser.add_argument("--comet_overwrite_checkpoints", - action="store_true", - help="Comet: Overwrite existing model checkpoints.") - parser.add_argument("--comet_checkpoint_filename", - nargs="?", - type=str, - default="best.pt", - help=("Comet: Name of the checkpoint file to save to Comet." - "Set to 'all' to log all checkpoints.")) - parser.add_argument("--comet_log_batch_metrics", - action="store_true", - help="Comet: Set to log batch level training metrics.") - parser.add_argument("--comet_log_batch_interval", - type=int, - default=1, - help="Comet: Logging frequency for batch level training metrics.") - parser.add_argument("--comet_log_prediction_interval", - type=int, - default=1, - help=("Comet: How often to log predictions." - "Applied at batch level.")) - parser.add_argument("--comet_log_confusion_matrix", - action="store_true", - help="Comet: Log a Confusion Matrix for the validation dataset.") - parser.add_argument("--comet_log_predictions", - action="store_true", - help="Comet: Log Predictions on Images from the Validation Set") - parser.add_argument("--comet_log_per_class_metrics", - action="store_true", - help="Comet: Log evaluation metrics for each class in the Validation Set") - parser.add_argument("--comet_max_image_uploads", - type=int, - default=100, - help="Comet: Maximum number of images to log to Comet.") - parser.add_argument("--comet_upload_dataset", - nargs="?", - const=True, - default=False, - help=("Comet: Upload Dataset to Comet as an Artifact." - "Set to 'train', 'val' or 'test' to upload a single dataset.")) - parser.add_argument("--comet_artifact", type=str, help="Comet: Name of the Comet dataset Artifact to download.") parser.add_argument("--comet_optimizer_config", type=str, help="Comet: Path to a Comet Optimizer Config File.") parser.add_argument("--comet_optimizer_id", type=str, help="Comet: ID of the Comet Optimizer sweep.") parser.add_argument("--comet_optimizer_objective", type=str, help="Comet: Set to 'minimize' or 'maximize'.") From f39ad862e3e2c4f8faadedde88d86815fa72e381 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 10:45:43 +0000 Subject: [PATCH 33/41] set types for integer/float env variables --- utils/loggers/comet/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 979cdb8bec31..55a4183be3ff 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -43,11 +43,11 @@ # Evaluation Settings COMET_LOG_CONFUSION_MATRIX = os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true" COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "false").lower() == "true" -COMET_MAX_IMAGE_UPLOADS = os.getenv("COMET_MAX_IMAGE_UPLOADS", 100) +COMET_MAX_IMAGE_UPLOADS = int(os.getenv("COMET_MAX_IMAGE_UPLOADS", 100)) # Confusion Matrix Settings -CONF_THRES = os.getenv("CONF_THRES", 0.001) -IOU_THRES = os.getenv("IOU_THRES", 0.6) +CONF_THRES = float(os.getenv("CONF_THRES", 0.001)) +IOU_THRES = float(os.getenv("IOU_THRES", 0.6)) # Batch Logging Settings COMET_LOG_BATCH_METRICS = os.getenv("COMET_LOG_BATCH_METRICS", "false").lower() == "true" From 427819edd7b9c0d319835ba262fe2af39ebb0591 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 10:54:21 +0000 Subject: [PATCH 34/41] update README --- utils/loggers/comet/README.md | 194 +++++++--------------------------- 1 file changed, 38 insertions(+), 156 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index 37619be2a858..1a46e54af3f9 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -54,7 +54,7 @@ That's it! Comet will automatically log your hyperparameters, command line argum yolo-ui # Try out an Example! -Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/ca87319ab1684f659a0ea6c625ef33f5?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) +Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/a0e29e0e9b984e4a822db2a62d0cb357?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) Or better yet, try it out yourself in this Colab Notebook @@ -85,75 +85,20 @@ By default, Comet will log the following items Comet can be configured to log additional data either through command line flags passed to the training script or through environment variables. -Here is the full list of command line options for Comet - ```shell - --comet_mode COMET_MODE - Comet: Set whether to run Comet in online or offline mode. - --comet_save_model Comet: Set to save model checkpoints. - --comet_model_name COMET_MODEL_NAME - Comet: Set the name for the saved model. - --comet_overwrite_checkpoints - Comet: Overwrite existing model checkpoints. - --comet_checkpoint_filename [COMET_CHECKPOINT_FILENAME] - Comet: Name of the checkpoint file to save to Comet.Set to 'all' to - log all checkpoints. - --comet_log_batch_metrics - Comet: Set to log batch level training metrics. - --comet_log_batch_interval COMET_LOG_BATCH_INTERVAL - Comet: Logging frequency for batch level training metrics. - --comet_log_prediction_interval COMET_LOG_PREDICTION_INTERVAL - Comet: How often to log predictions.Applied at batch level. - --comet_log_confusion_matrix - Comet: Log a Confusion Matrix for the validation dataset. - --comet_log_predictions - Comet: Log Predictions on Images from the Validation Set - --comet_log_per_class_metrics - Comet: Log evaluation metrics for each class in the Validation Set - --comet_max_image_uploads COMET_MAX_IMAGE_UPLOADS - Comet: Maximum number of images to log to Comet. - --comet_upload_dataset [COMET_UPLOAD_DATASET] - Comet: Upload Dataset to Comet as an Artifact.Set to 'train', 'val' - or 'test' to upload a single dataset. - --comet_artifact COMET_ARTIFACT - Comet: Name of the Comet dataset Artifact to download. +export COMET_MODE=online # Set whether to run Comet in 'online' or 'offline' mode. Defaults to online +export COMET_MODEL_NAME= #Set the name for the saved model. Defaults to yolov5 +export COMET_LOG_CONFUSION_MATRIX=false # Set to disable logging a Comet Confusion Matrix. Defaults to true +export COMET_MAX_IMAGE_UPLOADS= # Controls how many total image predictions to log to Comet. Defaults to 100. +export COMET_LOG_PER_CLASS_METRICS=true # Set to log evaluation metrics for each detected class at the end of training. Defaults to false +export COMET_DEFAULT_CHECKPOINT_FILENAME= # Set this if you would like to resume training from a different checkpoint. Defaults to 'last.pt' +export COMET_LOG_BATCH_LEVEL_METRICS=true # Set this if you would like to log training metrics at the batch level. Defaults to false. ``` -Let's take a look at these options. - ## Logging Checkpoints with Comet -Logging Models to Comet is disabled by default. To enable it, pass the `comet_save_model` flag to the training script. - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights yolov5s.pt \ ---save-period 1 \ ---comet_save_model -``` - -You have a few options when it comes to saving model checkpoints to Comet. Here's how you can change the checkpointing behavior. - -### Change which checkpoint file is logged -By default, Comet will save the `best.pt` checkpoint. To change which file gets saved, use the `comet_checkpoint_filename` argument. - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights yolov5s.pt \ ---save-period 1 \ ---comet_save_model \ ---comet_checkpoint_filename "last.pt" # this defaults to "best.pt" -``` - -### Log all checkpoints files +Logging Models to Comet is disabled by default. To enable it, pass the `save-period` argument to the training script. This will save the +logged checkpoints to Comet based on the interval value provided by `save-period` ```shell python train.py \ @@ -162,73 +107,17 @@ python train.py \ --epochs 5 \ --data coco128.yaml \ --weights yolov5s.pt \ ---save-period 1 \ ---comet_save_model \ ---comet_checkpoint_filename "all" -``` - -### Overwrite checkpoint files - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights yolov5s.pt \ ---save-period 1 \ ---comet_save_model \ ---comet_checkpoint_filename "best.pt" \ ---comet_overwrite_checkpoint -``` - -## Using a saved Checkpoint - -Comet will log a Run Path for every run that you can use to download model weights and resume runs. A run path is a string with the following format `comet:////` - -You can find the run path in the `Others` tab in your Comet Experiment. - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights "comet://" \ -``` - -By default, Comet will download the most recent checkpoint, this can be configured by specifying the checkpoint filename. You can specify a checkpoint for a particular epoch -by using the `comet_checkpoint_filename` flag. - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights "comet://" \ ---comet_checkpoint_filename "last_epoch_2.pt" +--save-period 1 ``` ## Logging Model Predictions -You can log model predictions and the associated images using `comet_log_predictions`. Predictions can be visualized using Comet's Object Detection Custom Panel +You can log model predictions and the associated images by passing the `bbox_interval` command line argument. Predictions can be visualized using Comet's Object Detection Custom Panel -Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) - -```shell -python train.py \ ---img 640 \ ---batch 16 \ ---epochs 5 \ ---data coco128.yaml \ ---weights yolov5s.pt \ ---comet_log_predictions -``` +**Note:** The YOLOv5 validation dataloader will default to a batch size of 32, so you will have to set the logging frequency accordingly. -### Controlling the number of Prediction Images logged to Comet +Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1&ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) -When logging predictions from YOLOv5, Comet will log the images associated with each set of predictions. By default a maximum of 100 validation images are logged. You can increase or decrease this number using the `comet_max_images` flag. ```shell python train.py \ @@ -237,44 +126,39 @@ python train.py \ --epochs 5 \ --data coco128.yaml \ --weights yolov5s.pt \ ---comet_log_predictions \ ---comet_max_image_uploads 200 +--bbox_interval 1 ``` -### Controlling the frequency of Prediction Images logged to Comet - -By default, every batch from the validation set is logged to Comet when logging predictions. This can be configured via the `comet_log_prediction_interval` flag. For example, setting this parameter to `2` will log every 2nd batch of data from the validation set. +### Controlling the number of Prediction Images logged to Comet -**Note:** The YOLOv5 validation dataloader will default to a batch size of 32, so you will have to set the logging frequency accordingly. +When logging predictions from YOLOv5, Comet will log the images associated with each set of predictions. By default a maximum of 100 validation images are logged. You can increase or decrease this number using the `COMET_MAX_IMAGE_UPLOADS` environment variable. ```shell -python train.py \ +env COMET_MAX_IMAGE_UPLOADS=200 python train.py \ --img 640 \ --batch 16 \ --epochs 5 \ --data coco128.yaml \ --weights yolov5s.pt \ ---comet_log_predictions \ ---comet_log_prediction_interval 2 +--bbox_interval 1 ``` ### Logging Class Level Metrics -Use the `comet_log_per_class_metrics` to log mAP, precision, recall, f1 for each class. +Use the `COMET_LOG_PER_CLASS_METRICS` environment variable to log mAP, precision, recall, f1 for each class. ```shell -python train.py \ +env COMET_LOG_PER_CLASS_METRICS=true python train.py \ --img 640 \ --batch 16 \ --epochs 5 \ --data coco128.yaml \ ---weights yolov5s.pt \ ---comet_log_per_class_metrics +--weights yolov5s.pt ``` ## Uploading a Dataset to Comet Artifacts -If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration), you can do so using the `comet_upload_dataset` flag. +If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration), you can do so using the `upload_dataset` flag. The dataset be organized in the way described in the [YOLOv5 documentation](https://docs.ultralytics.com/tutorials/train-custom-datasets/#3-organize-directories). The dataset config `yaml` file must follow the same format as that of the `coco128.yaml` file. @@ -285,8 +169,9 @@ python train.py \ --epochs 5 \ --data coco128.yaml \ --weights yolov5s.pt \ ---comet_upload_dataset +--upload_dataset ``` + You can find the uploaded dataset in the Artifacts tab in your Comet Workspace artifact-1 @@ -298,16 +183,21 @@ Artifacts are versioned and also support adding metadata about the dataset. Come ### Using a saved Artifact -If you would like to use a dataset from Comet Artifacts, simply pass the `comet_artifact` flag to the training script, along with the artifact path. The artifact path is a string with the following format `/:` +If you would like to use a dataset from Comet Artifacts, set the `path` variable in your dataset `yaml` file to point to the following Artifact resource URL. + +``` +# contents of artifact.yaml file +path="comet:///:" +``` +Then pass this file to your training script in the following way ```shell python train.py \ --img 640 \ --batch 16 \ --epochs 5 \ ---data coco128.yaml \ ---weights yolov5s.pt \ ---comet_artifact "examples/yolov5-dataset:latest" +--data artifact.yaml \ +--weights yolov5s.pt ``` Artifacts also allow you to track the lineage of data as it flows through your Experimentation workflow. Here you can see a graph that shows you all the experiments that have used your uploaded dataset. @@ -339,13 +229,14 @@ python utils/loggers/comet/hpo.py \ --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" ``` -The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional Comet related arguments to your sweep simply add them after +The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional arguments to your sweep simply add them after +the script. ```shell python utils/loggers/comet/hpo.py \ --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" \ - --comet_save_model \ - --comet_overwrite_checkpoints + --save-period 1 \ + --bbox_interval 1 ``` ### Running a Sweep in Parallel @@ -355,15 +246,6 @@ comet optimizer -j utils/loggers/comet/hpo.py \ utils/loggers/comet/optimizer_config.json" ``` -The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional Comet related arguments to your sweep simply add them after - -```shell -comet optimizer -j utils/loggers/comet/hpo.py \ - utils/loggers/comet/optimizer_config.json" \ - --comet_save_model \ - --comet_overwrite_checkpoints -``` - ### Visualizing Results Comet provides a number of ways to visualize the results of your sweep. Take a look at a [project with a completed sweep here](https://www.comet.com/examples/comet-example-yolov5/view/PrlArHGuuhDTKC1UuBmTtOSXD/panels?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) From 937bb7c29ce8040f0246bf8248cae7f261d132c2 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Sep 2022 13:21:34 +0000 Subject: [PATCH 35/41] fix typo in README --- utils/loggers/comet/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index 1a46e54af3f9..84c2e0088420 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -187,7 +187,7 @@ If you would like to use a dataset from Comet Artifacts, set the `path` variable ``` # contents of artifact.yaml file -path="comet:///:" +path: "comet:///:" ``` Then pass this file to your training script in the following way From f273392657d58613dbea5216684484c2fc876bad Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 6 Sep 2022 07:28:25 +0000 Subject: [PATCH 36/41] default to always logging model predictions --- utils/loggers/comet/README.md | 7 +++++-- utils/loggers/comet/__init__.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md index 84c2e0088420..7b0b8e0e2f09 100644 --- a/utils/loggers/comet/README.md +++ b/utils/loggers/comet/README.md @@ -93,6 +93,7 @@ export COMET_MAX_IMAGE_UPLOADS= # C export COMET_LOG_PER_CLASS_METRICS=true # Set to log evaluation metrics for each detected class at the end of training. Defaults to false export COMET_DEFAULT_CHECKPOINT_FILENAME= # Set this if you would like to resume training from a different checkpoint. Defaults to 'last.pt' export COMET_LOG_BATCH_LEVEL_METRICS=true # Set this if you would like to log training metrics at the batch level. Defaults to false. +export COMET_LOG_PREDICTIONS=true # Set this to false to disable logging model predictions ``` ## Logging Checkpoints with Comet @@ -112,7 +113,9 @@ python train.py \ ## Logging Model Predictions -You can log model predictions and the associated images by passing the `bbox_interval` command line argument. Predictions can be visualized using Comet's Object Detection Custom Panel +By default, model predictions (images, ground truth labels and bounding boxes) will be logged to Comet. + +You can control the frequency of logged predictions and the associated images by passing the `bbox_interval` command line argument. Predictions can be visualized using Comet's Object Detection Custom Panel. This frequency corresponds to every Nth batch of data per epoch. In the example below, we are logging every 2nd batch of data for each epoch. **Note:** The YOLOv5 validation dataloader will default to a batch size of 32, so you will have to set the logging frequency accordingly. @@ -126,7 +129,7 @@ python train.py \ --epochs 5 \ --data coco128.yaml \ --weights yolov5s.pt \ ---bbox_interval 1 +--bbox_interval 2 ``` ### Controlling the number of Prediction Images logged to Comet diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py index 55a4183be3ff..b168687dd7b2 100644 --- a/utils/loggers/comet/__init__.py +++ b/utils/loggers/comet/__init__.py @@ -42,7 +42,7 @@ # Evaluation Settings COMET_LOG_CONFUSION_MATRIX = os.getenv("COMET_LOG_CONFUSION_MATRIX", "true").lower() == "true" -COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "false").lower() == "true" +COMET_LOG_PREDICTIONS = os.getenv("COMET_LOG_PREDICTIONS", "true").lower() == "true" COMET_MAX_IMAGE_UPLOADS = int(os.getenv("COMET_MAX_IMAGE_UPLOADS", 100)) # Confusion Matrix Settings @@ -131,8 +131,12 @@ def __init__(self, opt, hyp, run_id=None, job_type="Training", **experiment_kwar else: self.iou_thres = IOU_THRES - self.comet_log_predictions = self.opt.bbox_interval > -1 - self.comet_log_prediction_interval = self.opt.bbox_interval + self.comet_log_predictions = COMET_LOG_PREDICTIONS + if self.opt.bbox_interval == -1: + self.comet_log_prediction_interval = self.opt.epochs // 10 if self.opt.epochs < 10 else 1 + else: + self.comet_log_prediction_interval = self.opt.bbox_interval + if self.comet_log_predictions: self.metadata_dict = {} @@ -428,7 +432,7 @@ def on_val_batch_start(self): return def on_val_batch_end(self, batch_i, images, targets, paths, shapes, outputs): - if not self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0): + if not (self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0)): return for si, pred in enumerate(outputs): From 0f97d4a7555c5a6ba363a2dc7e51540d9d1d48bd Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 7 Sep 2022 17:19:47 +0300 Subject: [PATCH 37/41] Update tutorial.ipynb --- tutorial.ipynb | 2042 +++++++++++++++++++++++------------------------- 1 file changed, 992 insertions(+), 1050 deletions(-) diff --git a/tutorial.ipynb b/tutorial.ipynb index d46529f27108..957437b2be6d 100644 --- a/tutorial.ipynb +++ b/tutorial.ipynb @@ -1,917 +1,217 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "t6MPjfT5NrKQ" - }, - "source": [ - "
\n", - "\n", - " \n", - " \n", - "\n", - "\n", - "
\n", - " \"Open\n", - " \"Open\n", - "
\n", - "\n", - "This YOLOv5 🚀 notebook by Ultralytics presents simple train, validate and predict examples to help start your AI adventure.
See GitHub for community support or contact us for professional support.\n", - "\n", - "
" - ] + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "YOLOv5 Tutorial", + "provenance": [], + "collapsed_sections": [], + "machine_shape": "hm", + "toc_visible": true }, - { - "cell_type": "markdown", - "metadata": { - "id": "7mGmQbAO5pQb" - }, - "source": [ - "# Setup\n", - "\n", - "Clone GitHub [repository](https://github.com/ultralytics/yolov5), install [dependencies](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) and check PyTorch and GPU." - ] + "kernelspec": { + "name": "python3", + "display_name": "Python 3" }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "accelerator": "GPU", + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "9b8caa3522fc4cbab31e13b5dfc7808d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_574140e4c4bc48c9a171541a02cd0211", + "IPY_MODEL_35e03ce5090346c9ae602891470fc555", + "IPY_MODEL_c942c208e72d46568b476bb0f2d75496" + ], + "layout": "IPY_MODEL_65881db1db8a4e9c930fab9172d45143" + } }, - "id": "wbvMlHd_QwMG", - "outputId": "0f9ee467-cea4-48e8-9050-7a76ae1b6141", - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n" - ] + "574140e4c4bc48c9a171541a02cd0211": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_60b913d755b34d638478e30705a2dde1", + "placeholder": "​", + "style": "IPY_MODEL_0856bea36ec148b68522ff9c9eb258d8", + "value": "100%" + } }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Setup complete ✅ (8 CPUs, 51.0 GB RAM, 37.4/166.8 GB disk)\n" - ] - } - ], - "source": [ - "!git clone https://github.com/ultralytics/yolov5 # clone\n", - "%cd yolov5\n", - "%pip install -qr requirements.txt # install\n", - "\n", - "import torch\n", - "import utils\n", - "display = utils.notebook_init() # checks" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4JnkELT0cIJg" - }, - "source": [ - "# 1. Detect\n", - "\n", - "`detect.py` runs YOLOv5 inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and saving results to `runs/detect`. Example inference sources are:\n", - "\n", - "```shell\n", - "python detect.py --source 0 # webcam\n", - " img.jpg # image \n", - " vid.mp4 # video\n", - " path/ # directory\n", - " 'path/*.jpg' # glob\n", - " 'https://youtu.be/Zgi9g1ksQHc' # YouTube\n", - " 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "35e03ce5090346c9ae602891470fc555": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76879f6f2aa54637a7a07faeea2bd684", + "max": 818322941, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_0ace3934ec6f4d36a1b3a9e086390926", + "value": 818322941 + } }, - "id": "zR9ZbuQCH7FX", - "outputId": "60647b99-e8d4-402c-f444-331bf6746da4", - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[34m\u001b[1mdetect: \u001b[0mweights=['yolov5s.pt'], source=data/images, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/detect, name=exp, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False\n", - "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", - "\n", - "Downloading https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt to yolov5s.pt...\n", - "100% 14.1M/14.1M [00:00<00:00, 27.8MB/s]\n", - "\n", - "Fusing layers... \n", - "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients\n", - "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 bus, 14.8ms\n", - "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 2 ties, 20.1ms\n", - "Speed: 0.6ms pre-process, 17.4ms inference, 21.6ms NMS per image at shape (1, 3, 640, 640)\n", - "Results saved to \u001b[1mruns/detect/exp\u001b[0m\n" - ] - } - ], - "source": [ - "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images\n", - "# display.Image(filename='runs/detect/exp/zidane.jpg', width=600)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hkAzDWJ7cWTr" - }, - "source": [ - "        \n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0eq1SMWl6Sfn" - }, - "source": [ - "# 2. Validate\n", - "Validate a model's accuracy on the [COCO](https://cocodataset.org/#home) dataset's `val` or `test` splits. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases). To show results by class use the `--verbose` flag." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 49, - "referenced_widgets": [ - "9b8caa3522fc4cbab31e13b5dfc7808d", - "574140e4c4bc48c9a171541a02cd0211", - "35e03ce5090346c9ae602891470fc555", - "c942c208e72d46568b476bb0f2d75496", - "65881db1db8a4e9c930fab9172d45143", - "60b913d755b34d638478e30705a2dde1", - "0856bea36ec148b68522ff9c9eb258d8", - "76879f6f2aa54637a7a07faeea2bd684", - "0ace3934ec6f4d36a1b3a9e086390926", - "d6b7a2243e0c4beca714d99dceec23d6", - "5966ba6e6f114d8c9d8d1d6b1bd4f4c7" - ] + "c942c208e72d46568b476bb0f2d75496": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6b7a2243e0c4beca714d99dceec23d6", + "placeholder": "​", + "style": "IPY_MODEL_5966ba6e6f114d8c9d8d1d6b1bd4f4c7", + "value": " 780M/780M [02:19<00:00, 6.24MB/s]" + } }, - "id": "WQPtK1QYVaD_", - "outputId": "102dabed-bc31-42fe-9133-d9ce28a2c01e", - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9b8caa3522fc4cbab31e13b5dfc7808d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0.00/780M [00:00

\n", - "Close the active learning loop by sampling images from your inference conditions with the `roboflow` pip package\n", - "

\n", - "\n", - "Train a YOLOv5s model on the [COCO128](https://www.kaggle.com/ultralytics/coco128) dataset with `--data coco128.yaml`, starting from pretrained `--weights yolov5s.pt`, or from randomly initialized `--weights '' --cfg yolov5s.yaml`.\n", - "\n", - "- **Pretrained [Models](https://github.com/ultralytics/yolov5/tree/master/models)** are downloaded\n", - "automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases)\n", - "- **[Datasets](https://github.com/ultralytics/yolov5/tree/master/data)** available for autodownload include: [COCO](https://github.com/ultralytics/yolov5/blob/master/data/coco.yaml), [COCO128](https://github.com/ultralytics/yolov5/blob/master/data/coco128.yaml), [VOC](https://github.com/ultralytics/yolov5/blob/master/data/VOC.yaml), [Argoverse](https://github.com/ultralytics/yolov5/blob/master/data/Argoverse.yaml), [VisDrone](https://github.com/ultralytics/yolov5/blob/master/data/VisDrone.yaml), [GlobalWheat](https://github.com/ultralytics/yolov5/blob/master/data/GlobalWheat2020.yaml), [xView](https://github.com/ultralytics/yolov5/blob/master/data/xView.yaml), [Objects365](https://github.com/ultralytics/yolov5/blob/master/data/Objects365.yaml), [SKU-110K](https://github.com/ultralytics/yolov5/blob/master/data/SKU-110K.yaml).\n", - "- **Training Results** are saved to `runs/train/` with incrementing run directories, i.e. `runs/train/exp2`, `runs/train/exp3` etc.\n", - "

\n", - "\n", - "A **Mosaic Dataloader** is used for training which combines 4 images into 1 mosaic.\n", - "\n", - "## Train on Custom Data with Roboflow 🌟 NEW\n", - "\n", - "[Roboflow](https://roboflow.com/?ref=ultralytics) enables you to easily **organize, label, and prepare** a high quality dataset with your own custom data. Roboflow also makes it easy to establish an active learning pipeline, collaborate with your team on dataset improvement, and integrate directly into your model building workflow with the `roboflow` pip package.\n", - "\n", - "- Custom Training Example: [https://blog.roboflow.com/how-to-train-yolov5-on-a-custom-dataset/](https://blog.roboflow.com/how-to-train-yolov5-on-a-custom-dataset/?ref=ultralytics)\n", - "- Custom Training Notebook: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow-ai/yolov5-custom-training-tutorial/blob/main/yolov5-custom-training.ipynb)\n", - "
\n", - "\n", - "

Label images lightning fast (including with model-assisted labeling)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "i3oKtE4g-aNn", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "#@title Select YOLOv5 🚀 logger {run: 'auto'}\n", - "logger = 'TensorBoard' #@param ['TensorBoard', 'ClearML', 'W&B', 'Comet']\n", - "\n", - "if logger == 'TensorBoard':\n", - " %load_ext tensorboard\n", - " %tensorboard --logdir runs/train\n", - "elif logger == 'ClearML':\n", - " %pip install -q clearml && clearml-init\n", - "elif logger == 'W&B':\n", - " %pip install -q wandb\n", - " import wandb; wandb.login()\n", - "elif logger == 'Comet':\n", - " %pip install -q comet_ml\n", - " import comet_ml; comet_ml.init()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" + "60b913d755b34d638478e30705a2dde1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - "id": "1NcFxRcFdJ_O", - "outputId": "baa6d4be-3379-4aab-844a-d5a5396c0e49", - "vscode": { - "languageId": "python" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[34m\u001b[1mtrain: \u001b[0mweights=yolov5s.pt, cfg=, data=coco128.yaml, hyp=data/hyps/hyp.scratch-low.yaml, epochs=3, batch_size=16, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, noplots=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, optimizer=SGD, sync_bn=False, workers=8, project=runs/train, name=exp, exist_ok=False, quad=False, cos_lr=False, label_smoothing=0.0, patience=100, freeze=[0], save_period=-1, seed=0, local_rank=-1, entity=None, upload_dataset=False, bbox_interval=-1, artifact_alias=latest\n", - "\u001b[34m\u001b[1mgithub: \u001b[0mup to date with https://github.com/ultralytics/yolov5 ✅\n", - "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", - "\n", - "\u001b[34m\u001b[1mhyperparameters: \u001b[0mlr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=1.0, obj=1.0, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.0, copy_paste=0.0\n", - "\u001b[34m\u001b[1mWeights & Biases: \u001b[0mrun 'pip install wandb' to automatically track and visualize YOLOv5 🚀 runs in Weights & Biases\n", - "\u001b[34m\u001b[1mClearML: \u001b[0mrun 'pip install clearml' to automatically track, visualize and remotely train YOLOv5 🚀 in ClearML\n", - "\u001b[34m\u001b[1mTensorBoard: \u001b[0mStart with 'tensorboard --logdir runs/train', view at http://localhost:6006/\n", - "\n", - "Dataset not found ⚠️, missing paths ['/content/datasets/coco128/images/train2017']\n", - "Downloading https://ultralytics.com/assets/coco128.zip to coco128.zip...\n", - "100% 6.66M/6.66M [00:00<00:00, 41.1MB/s]\n", - "Dataset download success ✅ (0.8s), saved to \u001b[1m/content/datasets\u001b[0m\n", - "\n", - " from n params module arguments \n", - " 0 -1 1 3520 models.common.Conv [3, 32, 6, 2, 2] \n", - " 1 -1 1 18560 models.common.Conv [32, 64, 3, 2] \n", - " 2 -1 1 18816 models.common.C3 [64, 64, 1] \n", - " 3 -1 1 73984 models.common.Conv [64, 128, 3, 2] \n", - " 4 -1 2 115712 models.common.C3 [128, 128, 2] \n", - " 5 -1 1 295424 models.common.Conv [128, 256, 3, 2] \n", - " 6 -1 3 625152 models.common.C3 [256, 256, 3] \n", - " 7 -1 1 1180672 models.common.Conv [256, 512, 3, 2] \n", - " 8 -1 1 1182720 models.common.C3 [512, 512, 1] \n", - " 9 -1 1 656896 models.common.SPPF [512, 512, 5] \n", - " 10 -1 1 131584 models.common.Conv [512, 256, 1, 1] \n", - " 11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", - " 12 [-1, 6] 1 0 models.common.Concat [1] \n", - " 13 -1 1 361984 models.common.C3 [512, 256, 1, False] \n", - " 14 -1 1 33024 models.common.Conv [256, 128, 1, 1] \n", - " 15 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", - " 16 [-1, 4] 1 0 models.common.Concat [1] \n", - " 17 -1 1 90880 models.common.C3 [256, 128, 1, False] \n", - " 18 -1 1 147712 models.common.Conv [128, 128, 3, 2] \n", - " 19 [-1, 14] 1 0 models.common.Concat [1] \n", - " 20 -1 1 296448 models.common.C3 [256, 256, 1, False] \n", - " 21 -1 1 590336 models.common.Conv [256, 256, 3, 2] \n", - " 22 [-1, 10] 1 0 models.common.Concat [1] \n", - " 23 -1 1 1182720 models.common.C3 [512, 512, 1, False] \n", - " 24 [17, 20, 23] 1 229245 models.yolo.Detect [80, [[10, 13, 16, 30, 33, 23], [30, 61, 62, 45, 59, 119], [116, 90, 156, 198, 373, 326]], [128, 256, 512]]\n", - "Model summary: 270 layers, 7235389 parameters, 7235389 gradients, 16.6 GFLOPs\n", - "\n", - "Transferred 349/349 items from yolov5s.pt\n", - "\u001b[34m\u001b[1mAMP: \u001b[0mchecks passed ✅\n", - "\u001b[34m\u001b[1moptimizer:\u001b[0m SGD(lr=0.01) with parameter groups 57 weight(decay=0.0), 60 weight(decay=0.0005), 60 bias\n", - "\u001b[34m\u001b[1malbumentations: \u001b[0mBlur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01), CLAHE(p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))\n", - "\u001b[34m\u001b[1mtrain: \u001b[0mScanning '/content/datasets/coco128/labels/train2017' images and labels...128 found, 0 missing, 2 empty, 0 corrupt: 100% 128/128 [00:00<00:00, 9659.25it/s]\n", - "\u001b[34m\u001b[1mtrain: \u001b[0mNew cache created: /content/datasets/coco128/labels/train2017.cache\n", - "\u001b[34m\u001b[1mtrain: \u001b[0mCaching images (0.1GB ram): 100% 128/128 [00:00<00:00, 951.31it/s]\n", - "\u001b[34m\u001b[1mval: \u001b[0mScanning '/content/datasets/coco128/labels/train2017.cache' images and labels... 128 found, 0 missing, 2 empty, 0 corrupt: 100% 128/128 [00:00\n", - "```\n", - "\n", - "**Run the YOLO training script**\n", - "```shell\n", - "python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt\n", - "```\n", - "\n", - "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/comet). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration) \n", - "\n", - "Get started by trying out the Comet Colab Notebook:\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing)\n", - "\n", - "\"yolo-ui\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Lay2WsTjNJzP" - }, - "source": [ - "## ClearML Logging and Automation 🌟 NEW\n", - "\n", - "[ClearML](https://cutt.ly/yolov5-notebook-clearml) is completely integrated into YOLOv5 to track your experimentation, manage dataset versions and even remotely execute training runs. To enable ClearML (check cells above):\n", - "\n", - "- `pip install clearml`\n", - "- run `clearml-init` to connect to a ClearML server (**deploy your own [open-source server](https://github.com/allegroai/clearml-server)**, or use our [free hosted server](https://cutt.ly/yolov5-notebook-clearml))\n", - "\n", - "You'll get all the great expected features from an experiment manager: live updates, model upload, experiment comparison etc. but ClearML also tracks uncommitted changes and installed packages for example. Thanks to that ClearML Tasks (which is what we call experiments) are also reproducible on different machines! With only 1 extra line, we can schedule a YOLOv5 training task on a queue to be executed by any number of ClearML Agents (workers).\n", - "\n", - "You can use ClearML Data to version your dataset and then pass it to YOLOv5 simply using its unique ID. This will help you keep track of your data without adding extra hassle. Explore the [ClearML Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml) for details!\n", - "\n", - "\n", - "\"ClearML" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DLI1JmHU7B0l" - }, - "source": [ - "## Weights & Biases Logging\n", - "\n", - "[Weights & Biases](https://wandb.ai/site?utm_campaign=repo_yolo_notebook) (W&B) is integrated with YOLOv5 for real-time visualization and cloud logging of training runs. This allows for better run comparison and introspection, as well improved visibility and collaboration for teams. To enable W&B `pip install wandb`, and then train normally (you will be guided through setup on first use). \n", - "\n", - "During training you will see live updates at [https://wandb.ai/home](https://wandb.ai/home?utm_campaign=repo_yolo_notebook), and you can create and share detailed [Reports](https://wandb.ai/glenn-jocher/yolov5_tutorial/reports/YOLOv5-COCO128-Tutorial-Results--VmlldzozMDI5OTY) of your results. For more information see the [YOLOv5 Weights & Biases Tutorial](https://github.com/ultralytics/yolov5/issues/1289). \n", - "\n", - "\n", - "\"Weights" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-WPvRbS5Swl6" - }, - "source": [ - "## Local Logging\n", - "\n", - "Training results are automatically logged with [Tensorboard](https://www.tensorflow.org/tensorboard) and [CSV](https://github.com/ultralytics/yolov5/pull/4148) loggers to `runs/train`, with a new experiment directory created for each new training as `runs/train/exp2`, `runs/train/exp3`, etc.\n", - "\n", - "This directory contains train and val statistics, mosaics, labels, predictions and augmentated mosaics, as well as metrics and charts including precision-recall (PR) curves and confusion matrices. \n", - "\n", - "\"Local\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Zelyeqbyt3GD" - }, - "source": [ - "# Environments\n", - "\n", - "YOLOv5 may be run in any of the following up-to-date verified environments (with all dependencies including [CUDA](https://developer.nvidia.com/cuda)/[CUDNN](https://developer.nvidia.com/cudnn), [Python](https://www.python.org/) and [PyTorch](https://pytorch.org/) preinstalled):\n", - "\n", - "- **Google Colab and Kaggle** notebooks with free GPU: \"Open \"Open\n", - "- **Google Cloud** Deep Learning VM. See [GCP Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/GCP-Quickstart)\n", - "- **Amazon** Deep Learning AMI. See [AWS Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/AWS-Quickstart)\n", - "- **Docker Image**. See [Docker Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/Docker-Quickstart) \"Docker\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6Qu7Iesl0p54" - }, - "source": [ - "# Status\n", - "\n", - "![CI CPU testing](https://github.com/ultralytics/yolov5/workflows/CI%20CPU%20testing/badge.svg)\n", - "\n", - "If this badge is green, all [YOLOv5 GitHub Actions](https://github.com/ultralytics/yolov5/actions) Continuous Integration (CI) tests are currently passing. CI tests verify correct operation of YOLOv5 training ([train.py](https://github.com/ultralytics/yolov5/blob/master/train.py)), testing ([val.py](https://github.com/ultralytics/yolov5/blob/master/val.py)), inference ([detect.py](https://github.com/ultralytics/yolov5/blob/master/detect.py)) and export ([export.py](https://github.com/ultralytics/yolov5/blob/master/export.py)) on macOS, Windows, and Ubuntu every 24 hours and on every commit.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IEijrePND_2I" - }, - "source": [ - "# Appendix\n", - "\n", - "Additional content below for PyTorch Hub, CI, reproducing results, profiling speeds, VOC training, classification training and TensorRT example." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GMusP4OAxFu6", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "import torch\n", - "\n", - "# PyTorch Hub Model\n", - "model = torch.hub.load('ultralytics/yolov5', 'yolov5s') # or yolov5n - yolov5x6, custom\n", - "\n", - "# Images\n", - "img = 'https://ultralytics.com/images/zidane.jpg' # or file, Path, PIL, OpenCV, numpy, list\n", - "\n", - "# Inference\n", - "results = model(img)\n", - "\n", - "# Results\n", - "results.print() # or .show(), .save(), .crop(), .pandas(), etc." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "FGH0ZjkGjejy", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# YOLOv5 CI\n", - "%%shell\n", - "rm -rf runs # remove runs/\n", - "m=yolov5n # official weights\n", - "b=runs/train/exp/weights/best # best.pt checkpoint\n", - "python train.py --imgsz 64 --batch 32 --weights $m.pt --cfg $m.yaml --epochs 1 --device 0 # train\n", - "for d in 0 cpu; do # devices\n", - " for w in $m $b; do # weights\n", - " python val.py --imgsz 64 --batch 32 --weights $w.pt --device $d # val\n", - " python detect.py --imgsz 64 --weights $w.pt --device $d # detect\n", - " done\n", - "done\n", - "python hubconf.py --model $m # hub\n", - "python models/tf.py --weights $m.pt # build TF model\n", - "python models/yolo.py --cfg $m.yaml # build PyTorch model\n", - "python export.py --weights $m.pt --img 64 --include torchscript # export" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "mcKoSIK2WSzj", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# Reproduce\n", - "for x in (f'yolov5{x}' for x in 'nsmlx'):\n", - " !python val.py --weights {x}.pt --data coco.yaml --img 640 --task speed # speed\n", - " !python val.py --weights {x}.pt --data coco.yaml --img 640 --conf 0.001 --iou 0.65 # mAP" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "gogI-kwi3Tye", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# Profile\n", - "from utils.torch_utils import profile\n", - "\n", - "m1 = lambda x: x * torch.sigmoid(x)\n", - "m2 = torch.nn.SiLU()\n", - "results = profile(input=torch.randn(16, 3, 640, 640), ops=[m1, m2], n=100)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BSgFCAcMbk1R", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# VOC\n", - "for b, m in zip([64, 64, 64, 32, 16], [f'yolov5{x}' for x in 'nsmlx']): # batch, model\n", - " !python train.py --batch {b} --weights {m}.pt --data VOC.yaml --epochs 50 --img 512 --hyp hyp.VOC.yaml --project VOC --name {m} --cache" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "UWGH7H6yakVl", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# Classification train\n", - "for m in [*(f'yolov5{x}-cls.pt' for x in 'nsmlx'), 'resnet50.pt', 'resnet101.pt', 'efficientnet_b0.pt', 'efficientnet_b1.pt']:\n", - " for d in 'mnist', 'fashion-mnist', 'cifar10', 'cifar100', 'imagenette160', 'imagenette320', 'imagenette', 'imagewoof160', 'imagewoof320', 'imagewoof':\n", - " !python classify/train.py --model {m} --data {d} --epochs 10 --project YOLOv5-cls --name {m}-{d}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "yYgOiFNHZx-1", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# Classification val\n", - "!bash data/scripts/get_imagenet.sh --val # download ImageNet val split (6.3G - 50000 images)\n", - "!python classify/val.py --weights yolov5m-cls.pt --data ../datasets/imagenet --img 224 # validate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "aq4DPWGu0Bl1", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# Validate on COCO test. Zip results.json and submit to eval server at https://competitions.codalab.org/competitions/20794\n", - "!bash data/scripts/get_coco.sh --test # download COCO test-dev2017 (7G - 40000 images, test 20000)\n", - "!python val.py --weights yolov5x.pt --data coco.yaml --img 640 --iou 0.65 --half --task test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VTRwsvA9u7ln", - "vscode": { - "languageId": "python" - } - }, - "outputs": [], - "source": [ - "# TensorRT \n", - "!pip install -U nvidia-tensorrt --index-url https://pypi.ngc.nvidia.com # install\n", - "!python export.py --weights yolov5s.pt --include engine --imgsz 640 --device 0 # export\n", - "!python detect.py --weights yolov5s.engine --imgsz 640 --device 0 # inference" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "machine_shape": "hm", - "name": "YOLOv5 Tutorial", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { "0856bea36ec148b68522ff9c9eb258d8": { "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0ace3934ec6f4d36a1b3a9e086390926": { - "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "35e03ce5090346c9ae602891470fc555": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_76879f6f2aa54637a7a07faeea2bd684", - "max": 818322941, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_0ace3934ec6f4d36a1b3a9e086390926", - "value": 818322941 - } - }, - "574140e4c4bc48c9a171541a02cd0211": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_60b913d755b34d638478e30705a2dde1", - "placeholder": "​", - "style": "IPY_MODEL_0856bea36ec148b68522ff9c9eb258d8", - "value": "100%" - } - }, - "5966ba6e6f114d8c9d8d1d6b1bd4f4c7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -923,10 +223,10 @@ "description_width": "" } }, - "60b913d755b34d638478e30705a2dde1": { + "76879f6f2aa54637a7a07faeea2bd684": { "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", "model_name": "LayoutModel", + "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -975,10 +275,26 @@ "width": null } }, - "65881db1db8a4e9c930fab9172d45143": { + "0ace3934ec6f4d36a1b3a9e086390926": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d6b7a2243e0c4beca714d99dceec23d6": { "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", "model_name": "LayoutModel", + "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1027,156 +343,782 @@ "width": null } }, - "76879f6f2aa54637a7a07faeea2bd684": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + "5966ba6e6f114d8c9d8d1d6b1bd4f4c7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "t6MPjfT5NrKQ" + }, + "source": [ + "
\n", + "\n", + " \n", + " \n", + "\n", + "\n", + "
\n", + " \"Open\n", + " \"Open\n", + "
\n", + "\n", + "This YOLOv5 🚀 notebook by Ultralytics presents simple train, validate and predict examples to help start your AI adventure.
See GitHub for community support or contact us for professional support.\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7mGmQbAO5pQb" + }, + "source": [ + "# Setup\n", + "\n", + "Clone GitHub [repository](https://github.com/ultralytics/yolov5), install [dependencies](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) and check PyTorch and GPU." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "wbvMlHd_QwMG", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "0f9ee467-cea4-48e8-9050-7a76ae1b6141" + }, + "source": [ + "!git clone https://github.com/ultralytics/yolov5 # clone\n", + "%cd yolov5\n", + "%pip install -qr requirements.txt # install\n", + "\n", + "import torch\n", + "import utils\n", + "display = utils.notebook_init() # checks" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Setup complete ✅ (8 CPUs, 51.0 GB RAM, 37.4/166.8 GB disk)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4JnkELT0cIJg" + }, + "source": [ + "# 1. Detect\n", + "\n", + "`detect.py` runs YOLOv5 inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and saving results to `runs/detect`. Example inference sources are:\n", + "\n", + "```shell\n", + "python detect.py --source 0 # webcam\n", + " img.jpg # image \n", + " vid.mp4 # video\n", + " path/ # directory\n", + " 'path/*.jpg' # glob\n", + " 'https://youtu.be/Zgi9g1ksQHc' # YouTube\n", + " 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream\n", + "```" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zR9ZbuQCH7FX", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "60647b99-e8d4-402c-f444-331bf6746da4" + }, + "source": [ + "!python detect.py --weights yolov5s.pt --img 640 --conf 0.25 --source data/images\n", + "# display.Image(filename='runs/detect/exp/zidane.jpg', width=600)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[34m\u001b[1mdetect: \u001b[0mweights=['yolov5s.pt'], source=data/images, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/detect, name=exp, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False\n", + "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", + "\n", + "Downloading https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt to yolov5s.pt...\n", + "100% 14.1M/14.1M [00:00<00:00, 27.8MB/s]\n", + "\n", + "Fusing layers... \n", + "YOLOv5s summary: 213 layers, 7225885 parameters, 0 gradients\n", + "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 bus, 14.8ms\n", + "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 2 ties, 20.1ms\n", + "Speed: 0.6ms pre-process, 17.4ms inference, 21.6ms NMS per image at shape (1, 3, 640, 640)\n", + "Results saved to \u001b[1mruns/detect/exp\u001b[0m\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hkAzDWJ7cWTr" + }, + "source": [ + "        \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0eq1SMWl6Sfn" + }, + "source": [ + "# 2. Validate\n", + "Validate a model's accuracy on the [COCO](https://cocodataset.org/#home) dataset's `val` or `test` splits. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases). To show results by class use the `--verbose` flag." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WQPtK1QYVaD_", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 49, + "referenced_widgets": [ + "9b8caa3522fc4cbab31e13b5dfc7808d", + "574140e4c4bc48c9a171541a02cd0211", + "35e03ce5090346c9ae602891470fc555", + "c942c208e72d46568b476bb0f2d75496", + "65881db1db8a4e9c930fab9172d45143", + "60b913d755b34d638478e30705a2dde1", + "0856bea36ec148b68522ff9c9eb258d8", + "76879f6f2aa54637a7a07faeea2bd684", + "0ace3934ec6f4d36a1b3a9e086390926", + "d6b7a2243e0c4beca714d99dceec23d6", + "5966ba6e6f114d8c9d8d1d6b1bd4f4c7" + ] }, - "9b8caa3522fc4cbab31e13b5dfc7808d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_574140e4c4bc48c9a171541a02cd0211", - "IPY_MODEL_35e03ce5090346c9ae602891470fc555", - "IPY_MODEL_c942c208e72d46568b476bb0f2d75496" + "outputId": "102dabed-bc31-42fe-9133-d9ce28a2c01e" + }, + "source": [ + "# Download COCO val\n", + "torch.hub.download_url_to_file('https://ultralytics.com/assets/coco2017val.zip', 'tmp.zip') # download (780M - 5000 images)\n", + "!unzip -q tmp.zip -d ../datasets && rm tmp.zip # unzip" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + " 0%| | 0.00/780M [00:00

\n", + "Close the active learning loop by sampling images from your inference conditions with the `roboflow` pip package\n", + "

\n", + "\n", + "Train a YOLOv5s model on the [COCO128](https://www.kaggle.com/ultralytics/coco128) dataset with `--data coco128.yaml`, starting from pretrained `--weights yolov5s.pt`, or from randomly initialized `--weights '' --cfg yolov5s.yaml`.\n", + "\n", + "- **Pretrained [Models](https://github.com/ultralytics/yolov5/tree/master/models)** are downloaded\n", + "automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases)\n", + "- **[Datasets](https://github.com/ultralytics/yolov5/tree/master/data)** available for autodownload include: [COCO](https://github.com/ultralytics/yolov5/blob/master/data/coco.yaml), [COCO128](https://github.com/ultralytics/yolov5/blob/master/data/coco128.yaml), [VOC](https://github.com/ultralytics/yolov5/blob/master/data/VOC.yaml), [Argoverse](https://github.com/ultralytics/yolov5/blob/master/data/Argoverse.yaml), [VisDrone](https://github.com/ultralytics/yolov5/blob/master/data/VisDrone.yaml), [GlobalWheat](https://github.com/ultralytics/yolov5/blob/master/data/GlobalWheat2020.yaml), [xView](https://github.com/ultralytics/yolov5/blob/master/data/xView.yaml), [Objects365](https://github.com/ultralytics/yolov5/blob/master/data/Objects365.yaml), [SKU-110K](https://github.com/ultralytics/yolov5/blob/master/data/SKU-110K.yaml).\n", + "- **Training Results** are saved to `runs/train/` with incrementing run directories, i.e. `runs/train/exp2`, `runs/train/exp3` etc.\n", + "

\n", + "\n", + "A **Mosaic Dataloader** is used for training which combines 4 images into 1 mosaic.\n", + "\n", + "## Train on Custom Data with Roboflow 🌟 NEW\n", + "\n", + "[Roboflow](https://roboflow.com/?ref=ultralytics) enables you to easily **organize, label, and prepare** a high quality dataset with your own custom data. Roboflow also makes it easy to establish an active learning pipeline, collaborate with your team on dataset improvement, and integrate directly into your model building workflow with the `roboflow` pip package.\n", + "\n", + "- Custom Training Example: [https://blog.roboflow.com/how-to-train-yolov5-on-a-custom-dataset/](https://blog.roboflow.com/how-to-train-yolov5-on-a-custom-dataset/?ref=ultralytics)\n", + "- Custom Training Notebook: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow-ai/yolov5-custom-training-tutorial/blob/main/yolov5-custom-training.ipynb)\n", + "
\n", + "\n", + "

Label images lightning fast (including with model-assisted labeling)" + ] + }, + { + "cell_type": "code", + "source": [ + "#@title Select YOLOv5 🚀 logger {run: 'auto'}\n", + "logger = 'TensorBoard' #@param ['TensorBoard', 'Comet', 'ClearML', 'W&B']\n", + "\n", + "if logger == 'TensorBoard':\n", + " %load_ext tensorboard\n", + " %tensorboard --logdir runs/train\n", + "elif logger == 'Comet':\n", + " %pip install -q comet_ml\n", + " import comet_ml; comet_ml.init()\n", + "elif logger == 'ClearML':\n", + " %pip install -q clearml && clearml-init\n", + "elif logger == 'W&B':\n", + " %pip install -q wandb\n", + " import wandb; wandb.login()" + ], + "metadata": { + "id": "i3oKtE4g-aNn" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "1NcFxRcFdJ_O", + "colab": { + "base_uri": "https://localhost:8080/" }, - "d6b7a2243e0c4beca714d99dceec23d6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + "outputId": "baa6d4be-3379-4aab-844a-d5a5396c0e49" + }, + "source": [ + "# Train YOLOv5s on COCO128 for 3 epochs\n", + "!python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --cache" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[34m\u001b[1mtrain: \u001b[0mweights=yolov5s.pt, cfg=, data=coco128.yaml, hyp=data/hyps/hyp.scratch-low.yaml, epochs=3, batch_size=16, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, noplots=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, optimizer=SGD, sync_bn=False, workers=8, project=runs/train, name=exp, exist_ok=False, quad=False, cos_lr=False, label_smoothing=0.0, patience=100, freeze=[0], save_period=-1, seed=0, local_rank=-1, entity=None, upload_dataset=False, bbox_interval=-1, artifact_alias=latest\n", + "\u001b[34m\u001b[1mgithub: \u001b[0mup to date with https://github.com/ultralytics/yolov5 ✅\n", + "YOLOv5 🚀 v6.2-56-g30e674b Python-3.7.13 torch-1.12.1+cu113 CUDA:0 (Tesla V100-SXM2-16GB, 16160MiB)\n", + "\n", + "\u001b[34m\u001b[1mhyperparameters: \u001b[0mlr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=1.0, obj=1.0, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.0, copy_paste=0.0\n", + "\u001b[34m\u001b[1mWeights & Biases: \u001b[0mrun 'pip install wandb' to automatically track and visualize YOLOv5 🚀 runs in Weights & Biases\n", + "\u001b[34m\u001b[1mClearML: \u001b[0mrun 'pip install clearml' to automatically track, visualize and remotely train YOLOv5 🚀 in ClearML\n", + "\u001b[34m\u001b[1mTensorBoard: \u001b[0mStart with 'tensorboard --logdir runs/train', view at http://localhost:6006/\n", + "\n", + "Dataset not found ⚠️, missing paths ['/content/datasets/coco128/images/train2017']\n", + "Downloading https://ultralytics.com/assets/coco128.zip to coco128.zip...\n", + "100% 6.66M/6.66M [00:00<00:00, 41.1MB/s]\n", + "Dataset download success ✅ (0.8s), saved to \u001b[1m/content/datasets\u001b[0m\n", + "\n", + " from n params module arguments \n", + " 0 -1 1 3520 models.common.Conv [3, 32, 6, 2, 2] \n", + " 1 -1 1 18560 models.common.Conv [32, 64, 3, 2] \n", + " 2 -1 1 18816 models.common.C3 [64, 64, 1] \n", + " 3 -1 1 73984 models.common.Conv [64, 128, 3, 2] \n", + " 4 -1 2 115712 models.common.C3 [128, 128, 2] \n", + " 5 -1 1 295424 models.common.Conv [128, 256, 3, 2] \n", + " 6 -1 3 625152 models.common.C3 [256, 256, 3] \n", + " 7 -1 1 1180672 models.common.Conv [256, 512, 3, 2] \n", + " 8 -1 1 1182720 models.common.C3 [512, 512, 1] \n", + " 9 -1 1 656896 models.common.SPPF [512, 512, 5] \n", + " 10 -1 1 131584 models.common.Conv [512, 256, 1, 1] \n", + " 11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 12 [-1, 6] 1 0 models.common.Concat [1] \n", + " 13 -1 1 361984 models.common.C3 [512, 256, 1, False] \n", + " 14 -1 1 33024 models.common.Conv [256, 128, 1, 1] \n", + " 15 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 16 [-1, 4] 1 0 models.common.Concat [1] \n", + " 17 -1 1 90880 models.common.C3 [256, 128, 1, False] \n", + " 18 -1 1 147712 models.common.Conv [128, 128, 3, 2] \n", + " 19 [-1, 14] 1 0 models.common.Concat [1] \n", + " 20 -1 1 296448 models.common.C3 [256, 256, 1, False] \n", + " 21 -1 1 590336 models.common.Conv [256, 256, 3, 2] \n", + " 22 [-1, 10] 1 0 models.common.Concat [1] \n", + " 23 -1 1 1182720 models.common.C3 [512, 512, 1, False] \n", + " 24 [17, 20, 23] 1 229245 models.yolo.Detect [80, [[10, 13, 16, 30, 33, 23], [30, 61, 62, 45, 59, 119], [116, 90, 156, 198, 373, 326]], [128, 256, 512]]\n", + "Model summary: 270 layers, 7235389 parameters, 7235389 gradients, 16.6 GFLOPs\n", + "\n", + "Transferred 349/349 items from yolov5s.pt\n", + "\u001b[34m\u001b[1mAMP: \u001b[0mchecks passed ✅\n", + "\u001b[34m\u001b[1moptimizer:\u001b[0m SGD(lr=0.01) with parameter groups 57 weight(decay=0.0), 60 weight(decay=0.0005), 60 bias\n", + "\u001b[34m\u001b[1malbumentations: \u001b[0mBlur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01), CLAHE(p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mScanning '/content/datasets/coco128/labels/train2017' images and labels...128 found, 0 missing, 2 empty, 0 corrupt: 100% 128/128 [00:00<00:00, 9659.25it/s]\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mNew cache created: /content/datasets/coco128/labels/train2017.cache\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mCaching images (0.1GB ram): 100% 128/128 [00:00<00:00, 951.31it/s]\n", + "\u001b[34m\u001b[1mval: \u001b[0mScanning '/content/datasets/coco128/labels/train2017.cache' images and labels... 128 found, 0 missing, 2 empty, 0 corrupt: 100% 128/128 [00:00 # 2. paste API key\n", + "python train.py --img 640 --epochs 3 --data coco128.yaml --weights yolov5s.pt # 3. train\n", + "```\n", + "\n", + "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/comet). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration). Get started by trying out the Comet Colab Notebook:\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing)\n", + "\n", + "\"yolo-ui\"" + ], + "metadata": { + "id": "nWOsI5wJR1o3" + } + }, + { + "cell_type": "markdown", + "source": [ + "## ClearML Logging and Automation 🌟 NEW\n", + "\n", + "[ClearML](https://cutt.ly/yolov5-notebook-clearml) is completely integrated into YOLOv5 to track your experimentation, manage dataset versions and even remotely execute training runs. To enable ClearML (check cells above):\n", + "\n", + "- `pip install clearml`\n", + "- run `clearml-init` to connect to a ClearML server (**deploy your own [open-source server](https://github.com/allegroai/clearml-server)**, or use our [free hosted server](https://cutt.ly/yolov5-notebook-clearml))\n", + "\n", + "You'll get all the great expected features from an experiment manager: live updates, model upload, experiment comparison etc. but ClearML also tracks uncommitted changes and installed packages for example. Thanks to that ClearML Tasks (which is what we call experiments) are also reproducible on different machines! With only 1 extra line, we can schedule a YOLOv5 training task on a queue to be executed by any number of ClearML Agents (workers).\n", + "\n", + "You can use ClearML Data to version your dataset and then pass it to YOLOv5 simply using its unique ID. This will help you keep track of your data without adding extra hassle. Explore the [ClearML Tutorial](https://github.com/ultralytics/yolov5/tree/master/utils/loggers/clearml) for details!\n", + "\n", + "\n", + "\"ClearML" + ], + "metadata": { + "id": "Lay2WsTjNJzP" } + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DLI1JmHU7B0l" + }, + "source": [ + "## Weights & Biases Logging\n", + "\n", + "[Weights & Biases](https://wandb.ai/site?utm_campaign=repo_yolo_notebook) (W&B) is integrated with YOLOv5 for real-time visualization and cloud logging of training runs. This allows for better run comparison and introspection, as well improved visibility and collaboration for teams. To enable W&B `pip install wandb`, and then train normally (you will be guided through setup on first use). \n", + "\n", + "During training you will see live updates at [https://wandb.ai/home](https://wandb.ai/home?utm_campaign=repo_yolo_notebook), and you can create and share detailed [Reports](https://wandb.ai/glenn-jocher/yolov5_tutorial/reports/YOLOv5-COCO128-Tutorial-Results--VmlldzozMDI5OTY) of your results. For more information see the [YOLOv5 Weights & Biases Tutorial](https://github.com/ultralytics/yolov5/issues/1289). \n", + "\n", + "\n", + "\"Weights" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-WPvRbS5Swl6" + }, + "source": [ + "## Local Logging\n", + "\n", + "Training results are automatically logged with [Tensorboard](https://www.tensorflow.org/tensorboard) and [CSV](https://github.com/ultralytics/yolov5/pull/4148) loggers to `runs/train`, with a new experiment directory created for each new training as `runs/train/exp2`, `runs/train/exp3`, etc.\n", + "\n", + "This directory contains train and val statistics, mosaics, labels, predictions and augmentated mosaics, as well as metrics and charts including precision-recall (PR) curves and confusion matrices. \n", + "\n", + "\"Local\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zelyeqbyt3GD" + }, + "source": [ + "# Environments\n", + "\n", + "YOLOv5 may be run in any of the following up-to-date verified environments (with all dependencies including [CUDA](https://developer.nvidia.com/cuda)/[CUDNN](https://developer.nvidia.com/cudnn), [Python](https://www.python.org/) and [PyTorch](https://pytorch.org/) preinstalled):\n", + "\n", + "- **Google Colab and Kaggle** notebooks with free GPU: \"Open \"Open\n", + "- **Google Cloud** Deep Learning VM. See [GCP Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/GCP-Quickstart)\n", + "- **Amazon** Deep Learning AMI. See [AWS Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/AWS-Quickstart)\n", + "- **Docker Image**. See [Docker Quickstart Guide](https://github.com/ultralytics/yolov5/wiki/Docker-Quickstart) \"Docker\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6Qu7Iesl0p54" + }, + "source": [ + "# Status\n", + "\n", + "![CI CPU testing](https://github.com/ultralytics/yolov5/workflows/CI%20CPU%20testing/badge.svg)\n", + "\n", + "If this badge is green, all [YOLOv5 GitHub Actions](https://github.com/ultralytics/yolov5/actions) Continuous Integration (CI) tests are currently passing. CI tests verify correct operation of YOLOv5 training ([train.py](https://github.com/ultralytics/yolov5/blob/master/train.py)), testing ([val.py](https://github.com/ultralytics/yolov5/blob/master/val.py)), inference ([detect.py](https://github.com/ultralytics/yolov5/blob/master/detect.py)) and export ([export.py](https://github.com/ultralytics/yolov5/blob/master/export.py)) on macOS, Windows, and Ubuntu every 24 hours and on every commit.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IEijrePND_2I" + }, + "source": [ + "# Appendix\n", + "\n", + "Additional content below for PyTorch Hub, CI, reproducing results, profiling speeds, VOC training, classification training and TensorRT example." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GMusP4OAxFu6" + }, + "source": [ + "import torch\n", + "\n", + "# PyTorch Hub Model\n", + "model = torch.hub.load('ultralytics/yolov5', 'yolov5s') # or yolov5n - yolov5x6, custom\n", + "\n", + "# Images\n", + "img = 'https://ultralytics.com/images/zidane.jpg' # or file, Path, PIL, OpenCV, numpy, list\n", + "\n", + "# Inference\n", + "results = model(img)\n", + "\n", + "# Results\n", + "results.print() # or .show(), .save(), .crop(), .pandas(), etc." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "FGH0ZjkGjejy" + }, + "source": [ + "# YOLOv5 CI\n", + "%%shell\n", + "rm -rf runs # remove runs/\n", + "m=yolov5n # official weights\n", + "b=runs/train/exp/weights/best # best.pt checkpoint\n", + "python train.py --imgsz 64 --batch 32 --weights $m.pt --cfg $m.yaml --epochs 1 --device 0 # train\n", + "for d in 0 cpu; do # devices\n", + " for w in $m $b; do # weights\n", + " python val.py --imgsz 64 --batch 32 --weights $w.pt --device $d # val\n", + " python detect.py --imgsz 64 --weights $w.pt --device $d # detect\n", + " done\n", + "done\n", + "python hubconf.py --model $m # hub\n", + "python models/tf.py --weights $m.pt # build TF model\n", + "python models/yolo.py --cfg $m.yaml # build PyTorch model\n", + "python export.py --weights $m.pt --img 64 --include torchscript # export" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "mcKoSIK2WSzj" + }, + "source": [ + "# Reproduce\n", + "for x in (f'yolov5{x}' for x in 'nsmlx'):\n", + " !python val.py --weights {x}.pt --data coco.yaml --img 640 --task speed # speed\n", + " !python val.py --weights {x}.pt --data coco.yaml --img 640 --conf 0.001 --iou 0.65 # mAP" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "gogI-kwi3Tye" + }, + "source": [ + "# Profile\n", + "from utils.torch_utils import profile\n", + "\n", + "m1 = lambda x: x * torch.sigmoid(x)\n", + "m2 = torch.nn.SiLU()\n", + "results = profile(input=torch.randn(16, 3, 640, 640), ops=[m1, m2], n=100)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "BSgFCAcMbk1R" + }, + "source": [ + "# VOC\n", + "for b, m in zip([64, 64, 64, 32, 16], [f'yolov5{x}' for x in 'nsmlx']): # batch, model\n", + " !python train.py --batch {b} --weights {m}.pt --data VOC.yaml --epochs 50 --img 512 --hyp hyp.VOC.yaml --project VOC --name {m} --cache" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Classification train\n", + "for m in [*(f'yolov5{x}-cls.pt' for x in 'nsmlx'), 'resnet50.pt', 'resnet101.pt', 'efficientnet_b0.pt', 'efficientnet_b1.pt']:\n", + " for d in 'mnist', 'fashion-mnist', 'cifar10', 'cifar100', 'imagenette160', 'imagenette320', 'imagenette', 'imagewoof160', 'imagewoof320', 'imagewoof':\n", + " !python classify/train.py --model {m} --data {d} --epochs 10 --project YOLOv5-cls --name {m}-{d}" + ], + "metadata": { + "id": "UWGH7H6yakVl" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Classification val\n", + "!bash data/scripts/get_imagenet.sh --val # download ImageNet val split (6.3G - 50000 images)\n", + "!python classify/val.py --weights yolov5m-cls.pt --data ../datasets/imagenet --img 224 # validate" + ], + "metadata": { + "id": "yYgOiFNHZx-1" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Validate on COCO test. Zip results.json and submit to eval server at https://competitions.codalab.org/competitions/20794\n", + "!bash data/scripts/get_coco.sh --test # download COCO test-dev2017 (7G - 40000 images, test 20000)\n", + "!python val.py --weights yolov5x.pt --data coco.yaml --img 640 --iou 0.65 --half --task test" + ], + "metadata": { + "id": "aq4DPWGu0Bl1" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "VTRwsvA9u7ln" + }, + "source": [ + "# TensorRT \n", + "!pip install -U nvidia-tensorrt --index-url https://pypi.ngc.nvidia.com # install\n", + "!python export.py --weights yolov5s.pt --include engine --imgsz 640 --device 0 # export\n", + "!python detect.py --weights yolov5s.engine --imgsz 640 --device 0 # inference" + ], + "execution_count": null, + "outputs": [] } - }, - "nbformat": 4, - "nbformat_minor": 0 + ] } From b7a210b94fa86d9b2b89d0277514c1c526d5653f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 7 Sep 2022 17:33:41 +0300 Subject: [PATCH 38/41] Update train.py --- train.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 2344abb46240..e16c17c499f0 100644 --- a/train.py +++ b/train.py @@ -482,9 +482,8 @@ def main(opt, callbacks=Callbacks()): check_git_status() check_requirements() - # Resume - if opt.resume and not check_wandb_resume(opt) and not check_comet_resume( - opt) or opt.evolve: # resume from specified or most recent last.pt + # Resume (from specified or most recent last.pt) + if opt.resume and not check_wandb_resume(opt) and not check_comet_resume(opt) or opt.evolve: last = Path(check_file(opt.resume) if isinstance(opt.resume, str) else get_latest_run()) opt_yaml = last.parent.parent / 'opt.yaml' # train options yaml opt_data = opt.data # original dataset From c01efb62d2671534408927f44adf32067ab8c871 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 7 Sep 2022 18:09:36 +0300 Subject: [PATCH 39/41] Add Comet to Integrations table --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1d6b4e153d82..6dd1e75dee6c 100644 --- a/README.md +++ b/README.md @@ -183,23 +183,27 @@ Get started in seconds with our verified environments. Click each icon below for ##
Integrations
+
+ + + - + - + - +
-|Deci ⭐ NEW|ClearML ⭐ NEW|Roboflow|Weights & Biases -|:-:|:-:|:-:|:-:| -|Automatically compile and quantize YOLOv5 for better inference performance in one click at [Deci](https://bit.ly/yolov5-deci-platform)|Automatically track, visualize and even remotely train YOLOv5 using [ClearML](https://cutt.ly/yolov5-readme-clearml) (open-source!)|Label and export your custom datasets directly to YOLOv5 for training with [Roboflow](https://roboflow.com/?ref=ultralytics) |Automatically track and visualize all your YOLOv5 training runs in the cloud with [Weights & Biases](https://wandb.ai/site?utm_campaign=repo_yolo_readme) +|Comet ⭐ NEW|Deci ⭐ NEW|ClearML ⭐ NEW|Roboflow|Weights & Biases +|:-:|:-:|:-:|:-:|:-:| +|Visualize model metrics and predictions and upload models and datasets in realtime with [Comet](https://www.comet.com/site/?ref=yolov5&utm_source=yolov5&utm_medium=affilliate&utm_campaign=yolov5_comet_integration)|Automatically compile and quantize YOLOv5 for better inference performance in one click at [Deci](https://bit.ly/yolov5-deci-platform)|Automatically track, visualize and even remotely train YOLOv5 using [ClearML](https://cutt.ly/yolov5-readme-clearml) (open-source!)|Label and export your custom datasets directly to YOLOv5 for training with [Roboflow](https://roboflow.com/?ref=ultralytics) |Automatically track and visualize all your YOLOv5 training runs in the cloud with [Weights & Biases](https://wandb.ai/site?utm_campaign=repo_yolo_readme) ##
Why YOLOv5
From c976c6457ec91fb09e35c6ef3db29cf77e2681a3 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 7 Sep 2022 18:19:56 +0300 Subject: [PATCH 40/41] Update README.md --- README.md | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6dd1e75dee6c..be799011a1a5 100644 --- a/README.md +++ b/README.md @@ -160,29 +160,10 @@ python train.py --data coco.yaml --cfg yolov5n.yaml --weights '' --batch-size 12 -##
Environments
- -Get started in seconds with our verified environments. Click each icon below for details. - -
- - - - - - - - - - - - - - -
##
Integrations
+
@@ -326,6 +307,28 @@ python export.py --weights yolov5s-cls.pt resnet50.pt efficientnet_b0.pt --inclu ``` + +##
Environments
+ +Get started in seconds with our verified environments. Click each icon below for details. + +
+ + + + + + + + + + + + + + +
+ ##
Contribute
From 1b33dd65553ffc86c56fb7a25feab11cf9633bf3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 15:20:42 +0000 Subject: [PATCH 41/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be799011a1a5..7763d174f92b 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ python export.py --weights yolov5s-cls.pt resnet50.pt efficientnet_b0.pt --inclu ``` - + ##
Environments
Get started in seconds with our verified environments. Click each icon below for details.