From fab241b8d038bc443230e0552efb70b7a90591f5 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 14 Jan 2021 20:43:51 -0800 Subject: [PATCH 01/20] W&B log epoch --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index f0ad206d4b94..2518f13dcbfa 100644 --- a/train.py +++ b/train.py @@ -361,7 +361,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}) # W&B + wandb.log({tag: x, 'epoch': epoch}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 5c7c44f10c3cda2b8d69255f5236ce1780c70188 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 14 Jan 2021 21:09:16 -0800 Subject: [PATCH 02/20] capitalize --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 2518f13dcbfa..af96a10b3586 100644 --- a/train.py +++ b/train.py @@ -361,7 +361,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x, 'epoch': epoch}) # W&B + wandb.log({tag: x, 'Epoch': epoch}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 09f80c3bd0a0ba2989b5c8398baefaf28f6da8b8 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 14 Jan 2021 20:43:51 -0800 Subject: [PATCH 03/20] W&B log epoch --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 9e6bd8673726..b533f4fede8a 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}) # W&B + wandb.log({tag: x, 'epoch': epoch}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 7521596ad6e0729d25bb23674b3738cc8327ef7a Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 14 Jan 2021 21:09:16 -0800 Subject: [PATCH 04/20] capitalize --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index b533f4fede8a..5897a1fd13ae 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x, 'epoch': epoch}) # W&B + wandb.log({tag: x, 'Epoch': epoch}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 97a202206d1462a0d51d25269737095382601a76 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 17:41:24 -0800 Subject: [PATCH 05/20] Update train.py New try using https://docs.wandb.ai/library/log#incremental-logging --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 5897a1fd13ae..a1356ebedff7 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x, 'Epoch': epoch}) # W&B + wandb.log({tag: x}, step=epoch) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 536e5ec602c1af1bd1d2c790e2fb0714aca08741 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 17:54:17 -0800 Subject: [PATCH 06/20] Update train.py --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index a1356ebedff7..ab4f8d738b8f 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}) + if x.exists()]}, step=epoch + 1) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch) # W&B + wandb.log({tag: x}, step=epoch + 1) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 51bcbc635a12672f9f8c79e29e672cc70c5b032d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:16:05 -0800 Subject: [PATCH 07/20] Update test.py --- test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index 891f6bef41c6..db344e722043 100644 --- a/test.py +++ b/test.py @@ -239,8 +239,8 @@ def test(data, if plots: confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: - wandb.log({"Images": wandb_images}) - wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))]}) + val_batches = [wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))] + wandb.log({"Images": wandb_images, "Validation": val_batches}, commit=False) # Save JSON if save_json and len(jdict): From 031fa84c9d759ade7a1e71cd17c9d0e63b0c87cf Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:20:22 -0800 Subject: [PATCH 08/20] Update train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index ab4f8d738b8f..30b8ea72e86a 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, step=epoch + 1) + if x.exists()]}, commit=False) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- From 23cfed96b538c410a9f9172ce398aaee3fd0b081 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:23:17 -0800 Subject: [PATCH 09/20] Update plots.py --- utils/plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/plots.py b/utils/plots.py index 4765069e0377..67f11bfd2011 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -295,7 +295,7 @@ def plot_labels(labels, save_dir=Path(''), loggers=None): # loggers for k, v in loggers.items() or {}: if k == 'wandb' and v: - v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}) + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() From b634d4b64f5865d06d17cfcf89c694b4f6476164 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:25:10 -0800 Subject: [PATCH 10/20] Update train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 30b8ea72e86a..f1db1b8a295a 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch + 1) # W&B + wandb.log({tag: x}, step=epoch) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From cf84ba93a728a6a6ca942b80ccdf84eb0b8594be Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:31:43 -0800 Subject: [PATCH 11/20] Update train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index f1db1b8a295a..b188d2461026 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, commit=False) + if x.exists()]}, step=epoch) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- From 820e5c664e75d43c4b1be9f0532f3d796144a947 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:36:57 -0800 Subject: [PATCH 12/20] label plot step -1 --- utils/plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/plots.py b/utils/plots.py index 67f11bfd2011..8326b3752ee4 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -295,7 +295,7 @@ def plot_labels(labels, save_dir=Path(''), loggers=None): # loggers for k, v in loggers.items() or {}: if k == 'wandb' and v: - v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, step=-1) def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() From e4096ff501491843d1e7071dfac10e81c18d0b91 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:46:25 -0800 Subject: [PATCH 13/20] update --- train.py | 2 +- utils/plots.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index b188d2461026..d36a047ae5d3 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch) # W&B + wandb.log({tag: x}, step=epoch, commit=True) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] diff --git a/utils/plots.py b/utils/plots.py index 8326b3752ee4..67f11bfd2011 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -295,7 +295,7 @@ def plot_labels(labels, save_dir=Path(''), loggers=None): # loggers for k, v in loggers.items() or {}: if k == 'wandb' and v: - v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, step=-1) + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() From a7a86bcc273571ab2cbd6a152d614b34c02880e6 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:49:40 -0800 Subject: [PATCH 14/20] update --- train.py | 2 +- utils/plots.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index d36a047ae5d3..b188d2461026 100644 --- a/train.py +++ b/train.py @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch, commit=True) # W&B + wandb.log({tag: x}, step=epoch) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] diff --git a/utils/plots.py b/utils/plots.py index 67f11bfd2011..fea2d02b8bbf 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -295,7 +295,7 @@ def plot_labels(labels, save_dir=Path(''), loggers=None): # loggers for k, v in loggers.items() or {}: if k == 'wandb' and v: - v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, step=0) def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() From d900efd8c86f8e2e67c9cbc376628fadb245b032 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:52:50 -0800 Subject: [PATCH 15/20] update --- train.py | 4 ++-- utils/plots.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index b188d2461026..5eff4bbac172 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, step=epoch) + if x.exists()]}, commit=False) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch) # W&B + wandb.log({tag: x}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] diff --git a/utils/plots.py b/utils/plots.py index fea2d02b8bbf..67f11bfd2011 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -295,7 +295,7 @@ def plot_labels(labels, save_dir=Path(''), loggers=None): # loggers for k, v in loggers.items() or {}: if k == 'wandb' and v: - v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, step=0) + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() From f00ab218b2e019de1a1e823a6b9b82d846cfbb62 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 18:59:59 -0800 Subject: [PATCH 16/20] update --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 5eff4bbac172..d8f1ae1a66a0 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, commit=False) + if x.exists()]}, step=epoch) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- From e039da70c5e6fb4e292ee477d2e0eb0bde3dccbc Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 19:08:37 -0800 Subject: [PATCH 17/20] update --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index d8f1ae1a66a0..f1db1b8a295a 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, step=epoch) + if x.exists()]}, commit=False) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}) # W&B + wandb.log({tag: x}, step=epoch) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 502f322ea480f6807dd7e4dc4e6eace04967161d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 19:52:48 -0800 Subject: [PATCH 18/20] update --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index f1db1b8a295a..b188d2461026 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, commit=False) + if x.exists()]}, step=epoch) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- From 1530981c698d674dd3609ca3de5f35b4666be33b Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 21:12:32 -0800 Subject: [PATCH 19/20] Update train.py --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index b188d2461026..9e6bd8673726 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}, step=epoch) + if x.exists()]}) # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- @@ -364,7 +364,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if tb_writer: tb_writer.add_scalar(tag, x, epoch) # tensorboard if wandb: - wandb.log({tag: x}, step=epoch) # W&B + wandb.log({tag: x}) # W&B # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] From 681356fbd5a02cceb02d793132ee8f60854e874c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 26 Jan 2021 21:14:39 -0800 Subject: [PATCH 20/20] Update train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 9e6bd8673726..5eff4bbac172 100644 --- a/train.py +++ b/train.py @@ -321,7 +321,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): # tb_writer.add_graph(model, imgs) # add model to tensorboard elif plots and ni == 10 and wandb: wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') - if x.exists()]}) + if x.exists()]}, commit=False) # end batch ------------------------------------------------------------------------------------------------ # end epoch ----------------------------------------------------------------------------------------------------