From 2737e6cb05b9748407bcb042689709d4417f39e3 Mon Sep 17 00:00:00 2001 From: ayush chaurasia Date: Thu, 11 Nov 2021 16:01:29 +0530 Subject: [PATCH 1/7] handle exceptions| attempt CI --- .github/workflows/ci-testing.yml | 6 +++++- utils/loggers/wandb/log_dataset.py | 4 +++- utils/loggers/wandb/wandb_utils.py | 16 +++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index abfe21ef8726..562acbf36516 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -52,11 +52,15 @@ jobs: python -m pip install --upgrade pip pip install -qr requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html pip install -q onnx tensorflow-cpu keras==2.6.0 # for export + pip install wandb python --version pip --version pip list shell: bash - + + - name: wandb login + run: | + wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa - name: Download data run: | # curl -L -o tmp.zip https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip diff --git a/utils/loggers/wandb/log_dataset.py b/utils/loggers/wandb/log_dataset.py index 8447272cdb48..64af3fda4c5a 100644 --- a/utils/loggers/wandb/log_dataset.py +++ b/utils/loggers/wandb/log_dataset.py @@ -1,13 +1,15 @@ import argparse from wandb_utils import WandbLogger +from utils.general import LOGGER WANDB_ARTIFACT_PREFIX = 'wandb-artifact://' def create_dataset_artifact(opt): logger = WandbLogger(opt, None, job_type='Dataset Creation') # TODO: return value unused - + if not logger.wandb: + LOGGER.info("install wandb using `pip install wandb` to log the dataset") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/utils/loggers/wandb/wandb_utils.py b/utils/loggers/wandb/wandb_utils.py index a71bc6ce96d2..9e32ed7a333a 100644 --- a/utils/loggers/wandb/wandb_utils.py +++ b/utils/loggers/wandb/wandb_utils.py @@ -17,7 +17,7 @@ sys.path.append(str(ROOT)) # add ROOT to PATH from utils.datasets import LoadImagesAndLabels, img2label_paths -from utils.general import check_dataset, check_file +from utils.general import check_dataset, check_file, LOGGER try: import wandb @@ -203,7 +203,7 @@ def check_and_upload_dataset(self, opt): config_path = self.log_dataset_artifact(opt.data, opt.single_cls, 'YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem) - print("Created dataset config file ", config_path) + LOGGER.info(f"Created dataset config file {config_path}") with open(config_path, errors='ignore') as f: wandb_data_dict = yaml.safe_load(f) return wandb_data_dict @@ -316,7 +316,7 @@ def log_model(self, path, opt, epoch, fitness_score, best_model=False): model_artifact.add_file(str(path / 'last.pt'), name='last.pt') wandb.log_artifact(model_artifact, aliases=['latest', 'last', 'epoch ' + str(self.current_epoch), 'best' if best_model else '']) - print("Saving model artifact on epoch ", epoch + 1) + LOGGER.info(f"Saving model artifact on epoch {epoch + 1}") def log_dataset_artifact(self, data_file, single_cls, project, overwrite_config=False): """ @@ -368,7 +368,7 @@ def map_val_table_path(self): Useful for - referencing artifacts for evaluation. """ self.val_table_path_map = {} - print("Mapping dataset") + LOGGER.info("Mapping dataset") for i, data in enumerate(tqdm(self.val_table.data)): self.val_table_path_map[data[3]] = data[0] @@ -488,7 +488,13 @@ def end_epoch(self, best_result=False): with all_logging_disabled(): if self.bbox_media_panel_images: self.log_dict["BoundingBoxDebugger"] = self.bbox_media_panel_images - wandb.log(self.log_dict) + try: + wandb.log(self.log_dict) + except BaseException as e: + LOGGER.info(f"An error occurred in wandb logger. The training will proceed without interruption. More info\n{e}") + self.wandb_run.finish() + self.wandb_run = None + self.log_dict = {} self.bbox_media_panel_images = [] if self.result_artifact: From acfe1e5c3bd49ddaa98f385c73ee144277cbd0b1 Mon Sep 17 00:00:00 2001 From: ayush chaurasia Date: Thu, 11 Nov 2021 16:03:36 +0530 Subject: [PATCH 2/7] update --- .github/workflows/ci-testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 562acbf36516..01499e6baeb2 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -58,9 +58,9 @@ jobs: pip list shell: bash - - name: wandb login - run: | - wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa + - name: wandb login + run: | + wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa - name: Download data run: | # curl -L -o tmp.zip https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip From f679b621f903171bdad2dbab5042848c863fd525 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 11 Nov 2021 14:16:18 +0100 Subject: [PATCH 3/7] Pre-commit manual run --- .github/workflows/ci-testing.yml | 2 +- utils/loggers/wandb/log_dataset.py | 2 ++ utils/loggers/wandb/wandb_utils.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 01499e6baeb2..30a0a611db08 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -57,7 +57,7 @@ jobs: pip --version pip list shell: bash - + - name: wandb login run: | wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa diff --git a/utils/loggers/wandb/log_dataset.py b/utils/loggers/wandb/log_dataset.py index 64af3fda4c5a..06e81fb69307 100644 --- a/utils/loggers/wandb/log_dataset.py +++ b/utils/loggers/wandb/log_dataset.py @@ -1,6 +1,7 @@ import argparse from wandb_utils import WandbLogger + from utils.general import LOGGER WANDB_ARTIFACT_PREFIX = 'wandb-artifact://' @@ -11,6 +12,7 @@ def create_dataset_artifact(opt): if not logger.wandb: LOGGER.info("install wandb using `pip install wandb` to log the dataset") + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, default='data/coco128.yaml', help='data.yaml path') diff --git a/utils/loggers/wandb/wandb_utils.py b/utils/loggers/wandb/wandb_utils.py index 9e32ed7a333a..47757dd1a74e 100644 --- a/utils/loggers/wandb/wandb_utils.py +++ b/utils/loggers/wandb/wandb_utils.py @@ -17,7 +17,7 @@ sys.path.append(str(ROOT)) # add ROOT to PATH from utils.datasets import LoadImagesAndLabels, img2label_paths -from utils.general import check_dataset, check_file, LOGGER +from utils.general import LOGGER, check_dataset, check_file try: import wandb From f64e960c5adead11a474a0a67ded1bace687438e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 12 Nov 2021 14:56:44 +0100 Subject: [PATCH 4/7] yaml one-liner --- .github/workflows/ci-testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 30a0a611db08..29cd4147473e 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -58,9 +58,9 @@ jobs: pip list shell: bash - - name: wandb login - run: | - wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa + - name: W&B login + run: wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa + - name: Download data run: | # curl -L -o tmp.zip https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip From 546b830d1bfb4db3e4c640dc1bc6843eea041276 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 12 Nov 2021 15:00:11 +0100 Subject: [PATCH 5/7] Update ci-testing.yml --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 29cd4147473e..d92de6965123 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -60,7 +60,7 @@ jobs: - name: W&B login run: wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa - + - name: Download data run: | # curl -L -o tmp.zip https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip From bb377fd9a36364052653cbe5c020a91b96f643ea Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 13 Nov 2021 19:53:14 +0100 Subject: [PATCH 6/7] Comment W&B CI Leave as example for future separate CI --- .github/workflows/ci-testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index d92de6965123..fa630afff8d8 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -51,15 +51,15 @@ jobs: run: | python -m pip install --upgrade pip pip install -qr requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install -q onnx tensorflow-cpu keras==2.6.0 # for export + pip install -q onnx tensorflow-cpu keras==2.6.0 # wandb # extras pip install wandb python --version pip --version pip list shell: bash - - name: W&B login - run: wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa + # - name: W&B login + # run: wandb login 345011b3fb26dc8337fd9b20e53857c1d403f2aa - name: Download data run: | From c5a8ab42af738d4f3302cbf20d15e10c6363e060 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 13 Nov 2021 19:53:35 +0100 Subject: [PATCH 7/7] Update ci-testing.yml --- .github/workflows/ci-testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index fa630afff8d8..5db6d41f4bcc 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -52,7 +52,6 @@ jobs: python -m pip install --upgrade pip pip install -qr requirements.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html pip install -q onnx tensorflow-cpu keras==2.6.0 # wandb # extras - pip install wandb python --version pip --version pip list