From 5748cb39f9f478159e506ccb61b366d02a267a5e Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 6 Apr 2022 08:59:24 -0700 Subject: [PATCH 1/2] Copy wandb param dict before training to avoid overwrites. Copy the hyperparameter dict retrieved from wandb configuration before passing it to `train()`. Training overwrites parameters in the dictionary (eg scaling obj/box/cls gains), which causes the values reported in wandb to not match the input values. This is confusing as it makes it hard to reproduce a run, and also throws off wandb's Bayesian sweep algorithm. --- utils/loggers/wandb/sweep.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/loggers/wandb/sweep.py b/utils/loggers/wandb/sweep.py index 206059bc30bf..c42ad794c5f5 100644 --- a/utils/loggers/wandb/sweep.py +++ b/utils/loggers/wandb/sweep.py @@ -16,8 +16,9 @@ def sweep(): wandb.init() - # Get hyp dict from sweep agent - hyp_dict = vars(wandb.config).get("_items") + # Get hyp dict from sweep agent. Copy it because train() will + # overwrite some parameters which confuses wandb. + hyp_dict = vars(wandb.config).get("_items").copy() # Workaround: get necessary opt args opt = parse_opt(known=True) From 323db2806f0bf6900df06ddc7dd3511d17d7f45e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 6 Apr 2022 18:24:01 +0200 Subject: [PATCH 2/2] Cleanup --- utils/loggers/wandb/sweep.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/loggers/wandb/sweep.py b/utils/loggers/wandb/sweep.py index c42ad794c5f5..d49ea6f2778b 100644 --- a/utils/loggers/wandb/sweep.py +++ b/utils/loggers/wandb/sweep.py @@ -16,8 +16,7 @@ def sweep(): wandb.init() - # Get hyp dict from sweep agent. Copy it because train() will - # overwrite some parameters which confuses wandb. + # Get hyp dict from sweep agent. Copy because train() modifies parameters which confused wandb. hyp_dict = vars(wandb.config).get("_items").copy() # Workaround: get necessary opt args