From 530a4394c3617a23390fcd72970bf0fd8c7c3317 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 14:12:07 +0200 Subject: [PATCH 01/12] Update evolution to CSV format --- train.py | 21 ++++++++++++--------- utils/general.py | 22 +++++++++++----------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/train.py b/train.py index 34bd8e73c290..591edcf5eb3f 100644 --- a/train.py +++ b/train.py @@ -464,7 +464,7 @@ def main(opt): check_requirements(requirements=FILE.parent / 'requirements.txt', exclude=['thop']) # Resume - if opt.resume and not check_wandb_resume(opt): # resume an interrupted run + if opt.resume and not check_wandb_resume(opt) and not opt.evolve: # resume an interrupted run ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' with open(Path(ckpt).parent.parent / 'opt.yaml') as f: @@ -474,8 +474,10 @@ def main(opt): else: opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' - opt.name = 'evolve' if opt.evolve else opt.name - opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve)) + if opt.evolve: + opt.project = 'runs/evolve' + opt.exist_ok = opt.resume + opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok) # DDP mode device = select_device(opt.device, batch_size=opt.batch_size) @@ -535,15 +537,15 @@ def main(opt): hyp['anchors'] = 3 opt.noval, opt.nosave = True, True # only val/save final epoch # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices - yaml_file = Path(opt.save_dir) / 'hyp_evolved.yaml' # save best result here + yaml_file, evolve_csv = opt.save_dir / 'hyp_evolved.yaml', opt.save_dir / 'evolve.csv' if opt.bucket: - os.system(f'gsutil cp gs://{opt.bucket}/evolve.txt .') # download evolve.txt if exists + os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {opt.save_dir}') # download evolve.csv if exists for _ in range(opt.evolve): # generations to evolve - if Path('evolve.txt').exists(): # if evolve.txt exists: select best hyps and mutate + if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate # Select parent(s) parent = 'single' # parent selection method: 'single' or 'weighted' - x = np.loadtxt('evolve.txt', ndmin=2) + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',') n = min(5, len(x)) # number of previous results to consider x = x[np.argsort(-fitness(x))][:n] # top n mutations w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) @@ -575,11 +577,12 @@ def main(opt): results = train(hyp.copy(), opt, device) # Write mutation results - print_mutation(hyp.copy(), results, yaml_file, opt.bucket) + print_mutation(hyp.copy(), results, yaml_file, evolve_csv, opt.bucket) # Plot results plot_evolution(yaml_file) - print(f'Hyperparameter evolution complete. Best results saved as: {yaml_file}\n' + print(f'Hyperparameter evolution complete.\n' + f'Best results saved as: {yaml_file}\n' f'Command to train a new model with these hyperparameters: $ python train.py --hyp {yaml_file}') diff --git a/utils/general.py b/utils/general.py index ed028d2b3765..8b1602e3bd4d 100755 --- a/utils/general.py +++ b/utils/general.py @@ -615,23 +615,23 @@ def strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_op print(f"Optimizer stripped from {f},{(' saved as %s,' % s) if s else ''} {mb:.1f}MB") -def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''): - # Print mutation results to evolve.txt (for use with train.py --evolve) +def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', csv='results.csv', bucket=''): + # Print mutation results to evolve.csv (for use with train.py --evolve) a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) - print('\n%s\n%s\nEvolved fitness: %s\n' % (a, b, c)) + print(f'\n{a}\n{b}\nEvolved fitness: {c}\n') if bucket: - url = 'gs://%s/evolve.txt' % bucket - if gsutil_getsize(url) > (os.path.getsize('evolve.txt') if os.path.exists('evolve.txt') else 0): - os.system('gsutil cp %s .' % url) # download evolve.txt if larger than local + url = f'gs://{bucket}/evolve.csv' + if gsutil_getsize(url) > (os.path.getsize('evolve.csv') if os.path.exists('evolve.csv') else 0): + os.system(f'gsutil cp {url} .') # download evolve.csv if larger than local - with open('evolve.txt', 'a') as f: # append result + with open(csv, 'a') as f: # append result f.write(c + b + '\n') - x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows + x = np.unique(np.loadtxt(csv, ndmin=2), axis=0) # load unique rows x = x[np.argsort(-fitness(x))] # sort - np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness + np.savetxt(csv, x, '%10.3g') # save sort by fitness # Save yaml for i, k in enumerate(hyp.keys()): @@ -639,11 +639,11 @@ def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''): with open(yaml_file, 'w') as f: results = tuple(x[0, :7]) c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) - f.write('# Hyperparameter Evolution Results\n# Generations: %g\n# Metrics: ' % len(x) + c + '\n\n') + f.write(f'# Hyperparameter Evolution Results\n# Generations: {len(x)}\n# Metrics: {c}\n\n') yaml.safe_dump(hyp, f, sort_keys=False) if bucket: - os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload + os.system(f'gsutil cp evolve.csv {yaml_file} gs://{bucket}') # upload def apply_classifier(x, model, img, im0): From 43e325f30b822c695544afd79b3b7fbc06c5dc1a Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 14:14:14 +0200 Subject: [PATCH 02/12] Update --- train.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/train.py b/train.py index 591edcf5eb3f..02da22172354 100644 --- a/train.py +++ b/train.py @@ -477,7 +477,7 @@ def main(opt): if opt.evolve: opt.project = 'runs/evolve' opt.exist_ok = opt.resume - opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok) + opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # DDP mode device = select_device(opt.device, batch_size=opt.batch_size) @@ -535,11 +535,11 @@ def main(opt): hyp = yaml.safe_load(f) # load hyps dict if 'anchors' not in hyp: # anchors commented in hyp.yaml hyp['anchors'] = 3 - opt.noval, opt.nosave = True, True # only val/save final epoch + opt.noval, opt.nosave, save_dir = True, True, Path(opt.save_dir) # only val/save final epoch # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices - yaml_file, evolve_csv = opt.save_dir / 'hyp_evolved.yaml', opt.save_dir / 'evolve.csv' + yaml_file, evolve_csv = save_dir / 'hyp_evolved.yaml', save_dir / 'evolve.csv' if opt.bucket: - os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {opt.save_dir}') # download evolve.csv if exists + os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {save_dir}') # download evolve.csv if exists for _ in range(opt.evolve): # generations to evolve if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate From 74e936b630e59ce09664e004782b5f4fb32ce175 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 15:51:14 +0200 Subject: [PATCH 03/12] Update --- train.py | 14 +++++++----- utils/general.py | 48 +++++++++++++++++++++++---------------- utils/loggers/__init__.py | 3 +-- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/train.py b/train.py index 02da22172354..62a2e206615b 100644 --- a/train.py +++ b/train.py @@ -367,7 +367,8 @@ def train(hyp, # path/to/hyp.yaml or hyp dictionary fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] if fi > best_fitness: best_fitness = fi - callbacks.on_fit_epoch_end(mloss, results, lr, epoch, best_fitness, fi) + log_vals = list(mloss) + list(results) + lr + callbacks.on_fit_epoch_end(log_vals, epoch, best_fitness, fi) # Save model if (not nosave) or (final_epoch and not evolve): # if save @@ -420,9 +421,9 @@ def parse_opt(known=False): parser.add_argument('--cfg', type=str, default='', help='model.yaml path') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='dataset.yaml path') parser.add_argument('--hyp', type=str, default='data/hyps/hyp.scratch.yaml', help='hyperparameters path') - parser.add_argument('--epochs', type=int, default=300) + parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') - parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=128, help='train, val image size (pixels)') parser.add_argument('--rect', action='store_true', help='rectangular training') parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') @@ -437,7 +438,7 @@ def parse_opt(known=False): parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') - parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') + parser.add_argument('--workers', type=int, default=0, help='maximum number of dataloader workers') parser.add_argument('--project', default='runs/train', help='save to project/name') parser.add_argument('--entity', default=None, help='W&B entity') parser.add_argument('--name', default='exp', help='save to project/name') @@ -452,6 +453,7 @@ def parse_opt(known=False): parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') parser.add_argument('--freeze', type=int, default=0, help='Number of layers to freeze. backbone=10, all=24') opt = parser.parse_known_args()[0] if known else parser.parse_args() + opt.evolve = 2 return opt @@ -545,7 +547,7 @@ def main(opt): if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate # Select parent(s) parent = 'single' # parent selection method: 'single' or 'weighted' - x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',') + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=(',')) n = min(5, len(x)) # number of previous results to consider x = x[np.argsort(-fitness(x))][:n] # top n mutations w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) @@ -577,7 +579,7 @@ def main(opt): results = train(hyp.copy(), opt, device) # Write mutation results - print_mutation(hyp.copy(), results, yaml_file, evolve_csv, opt.bucket) + print_mutation(results, hyp.copy(), save_dir, opt.bucket) # Plot results plot_evolution(yaml_file) diff --git a/utils/general.py b/utils/general.py index 8b1602e3bd4d..b1936841bcf6 100755 --- a/utils/general.py +++ b/utils/general.py @@ -615,35 +615,43 @@ def strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_op print(f"Optimizer stripped from {f},{(' saved as %s,' % s) if s else ''} {mb:.1f}MB") -def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', csv='results.csv', bucket=''): - # Print mutation results to evolve.csv (for use with train.py --evolve) - a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys - b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values - c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) - print(f'\n{a}\n{b}\nEvolved fitness: {c}\n') +def print_mutation(results, hyp, save_dir, bucket): + evolve_csv, results_csv, evolve_yaml = save_dir / 'evolve.csv', save_dir / 'results.csv', save_dir / 'hyp_evolve.yaml' + keys = ('metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', + 'val/box_loss', 'val/obj_loss', 'val/cls_loss') + tuple(hyp.keys()) # [results + hyps] + keys = tuple(x.strip() for x in keys) + vals = results + tuple(hyp.values()) + n = len(keys) + # Download (optional) if bucket: url = f'gs://{bucket}/evolve.csv' - if gsutil_getsize(url) > (os.path.getsize('evolve.csv') if os.path.exists('evolve.csv') else 0): - os.system(f'gsutil cp {url} .') # download evolve.csv if larger than local + if gsutil_getsize(url) > (os.path.getsize(evolve_csv) if os.path.exists(evolve_csv) else 0): + os.system(f'gsutil cp {url} {save_dir}') # download evolve.csv if larger than local + + # Log to evolve.csv + s = '' if evolve_csv.exists() else (('%20s,' * n % keys).rstrip(',') + '\n') # add header + with open(evolve_csv, 'a') as f: + f.write(s + ('%20.5g,' * n % vals).rstrip(',') + '\n') - with open(csv, 'a') as f: # append result - f.write(c + b + '\n') - x = np.unique(np.loadtxt(csv, ndmin=2), axis=0) # load unique rows - x = x[np.argsort(-fitness(x))] # sort - np.savetxt(csv, x, '%10.3g') # save sort by fitness + # Print to screen + print(colorstr('evolve: ') + ', '.join(f'{x.strip():>20s}' for x in keys)) + print(colorstr('evolve: ') + ', '.join(f'{x:20.5g}' for x in vals)) # Save yaml - for i, k in enumerate(hyp.keys()): - hyp[k] = float(x[0, i + 7]) - with open(yaml_file, 'w') as f: - results = tuple(x[0, :7]) - c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) - f.write(f'# Hyperparameter Evolution Results\n# Generations: {len(x)}\n# Metrics: {c}\n\n') + with open(evolve_yaml, 'w') as f: + data = pd.read_csv(evolve_csv) + data = data.rename(columns=lambda x: x.strip()) # strip keys + i = np.argmax(fitness(data.values[:, :7])) # + f.write(f'# YOLOv5 Hyperparameter Evolution Results\n' + + f'# Best generation: {i}\n' + + f'# Last generation: {len(data)}\n' + + f'# ' + ', '.join(f'{x.strip():>20s}' for x in keys[:7]) + '\n' + + f'# ' + ', '.join(f'{x:>20.5g}' for x in data.values[i, :7]) + '\n\n') yaml.safe_dump(hyp, f, sort_keys=False) if bucket: - os.system(f'gsutil cp evolve.csv {yaml_file} gs://{bucket}') # upload + os.system(f'gsutil cp {evolve_csv} {evolve_yaml} gs://{bucket}') # upload def apply_classifier(x, model, img, im0): diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index be76d0c17f1b..5315fb73b0c5 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -95,9 +95,8 @@ def on_val_end(self): files = sorted(self.save_dir.glob('val*.jpg')) self.wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in files]}) - def on_fit_epoch_end(self, mloss, results, lr, epoch, best_fitness, fi): + def on_fit_epoch_end(self, vals, epoch, best_fitness, fi): # Callback runs at the end of each fit (train+val) epoch - vals = list(mloss) + list(results) + lr x = {k: v for k, v in zip(self.keys, vals)} # dict if self.csv: file = self.save_dir / 'results.csv' From ae2c378887ca9c34000df4947c3ba5c590148233 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 15:53:14 +0200 Subject: [PATCH 04/12] Update --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 62a2e206615b..29d6a94a70e7 100644 --- a/train.py +++ b/train.py @@ -547,7 +547,7 @@ def main(opt): if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate # Select parent(s) parent = 'single' # parent selection method: 'single' or 'weighted' - x = np.loadtxt(evolve_csv, ndmin=2, delimiter=(',')) + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=(','), skiprows=1) n = min(5, len(x)) # number of previous results to consider x = x[np.argsort(-fitness(x))][:n] # top n mutations w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) From 6981cb4b6c2a36ff122028c200e584702e736d93 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:11:12 +0200 Subject: [PATCH 05/12] Update --- train.py | 12 ++++++------ utils/plots.py | 50 +++++++++++++++++++++++++------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/train.py b/train.py index 29d6a94a70e7..8b946c2e2954 100644 --- a/train.py +++ b/train.py @@ -37,7 +37,7 @@ check_requirements, print_mutation, set_logging, one_cycle, colorstr, methods from utils.downloads import attempt_download from utils.loss import ComputeLoss -from utils.plots import plot_labels, plot_evolution +from utils.plots import plot_labels, plot_evolve from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first, de_parallel from utils.loggers.wandb.wandb_utils import check_wandb_resume from utils.metrics import fitness @@ -539,7 +539,7 @@ def main(opt): hyp['anchors'] = 3 opt.noval, opt.nosave, save_dir = True, True, Path(opt.save_dir) # only val/save final epoch # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices - yaml_file, evolve_csv = save_dir / 'hyp_evolved.yaml', save_dir / 'evolve.csv' + evolve_yaml, evolve_csv = save_dir / 'hyp_evolved.yaml', save_dir / 'evolve.csv' if opt.bucket: os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {save_dir}') # download evolve.csv if exists @@ -547,7 +547,7 @@ def main(opt): if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate # Select parent(s) parent = 'single' # parent selection method: 'single' or 'weighted' - x = np.loadtxt(evolve_csv, ndmin=2, delimiter=(','), skiprows=1) + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',', skiprows=1) n = min(5, len(x)) # number of previous results to consider x = x[np.argsort(-fitness(x))][:n] # top n mutations w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) @@ -582,10 +582,10 @@ def main(opt): print_mutation(results, hyp.copy(), save_dir, opt.bucket) # Plot results - plot_evolution(yaml_file) + plot_evolve(evolve_csv) print(f'Hyperparameter evolution complete.\n' - f'Best results saved as: {yaml_file}\n' - f'Command to train a new model with these hyperparameters: $ python train.py --hyp {yaml_file}') + f'Best results saved as: {evolve_yaml}\n' + f'Command to train a new model with these hyperparameters: $ python train.py --hyp {evolve_yaml}') def run(**kwargs): diff --git a/utils/plots.py b/utils/plots.py index 252e128168ee..ef850ee2f26d 100644 --- a/utils/plots.py +++ b/utils/plots.py @@ -325,30 +325,6 @@ def plot_labels(labels, names=(), save_dir=Path('')): plt.close() -def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() - # Plot hyperparameter evolution results in evolve.txt - with open(yaml_file) as f: - hyp = yaml.safe_load(f) - x = np.loadtxt('evolve.txt', ndmin=2) - f = fitness(x) - # weights = (f - f.min()) ** 2 # for weighted results - plt.figure(figsize=(10, 12), tight_layout=True) - matplotlib.rc('font', **{'size': 8}) - for i, (k, v) in enumerate(hyp.items()): - y = x[:, i + 7] - # mu = (y * weights).sum() / weights.sum() # best weighted result - mu = y[f.argmax()] # best single result - plt.subplot(6, 5, i + 1) - plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none') - plt.plot(mu, f.max(), 'k+', markersize=15) - plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters - if i % 5 != 0: - plt.yticks([]) - print('%15s: %.3g' % (k, mu)) - plt.savefig('evolve.png', dpi=200) - print('\nPlot saved as evolve.png') - - def profile_idetection(start=0, stop=0, labels=(), save_dir=''): # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection() ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel() @@ -381,7 +357,31 @@ def profile_idetection(start=0, stop=0, labels=(), save_dir=''): plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200) -def plot_results(file='', dir=''): +def plot_evolve(evolve_csv=Path('path/to/evolve.csv')): # from utils.plots import *; plot_evolve() + # Plot evolve.csv hyp evolution results + data = pd.read_csv(evolve_csv) + keys = [x.strip() for x in data.columns] + x = data.values + f = fitness(x) + j = np.argmax(f) # max fitness index + plt.figure(figsize=(10, 12), tight_layout=True) + matplotlib.rc('font', **{'size': 8}) + for i, k in enumerate(keys[7:]): + v = x[:, 7 + i] + mu = v[j] # best single result + plt.subplot(6, 5, i + 1) + plt.scatter(v, f, c=hist2d(v, f, 20), cmap='viridis', alpha=.8, edgecolors='none') + plt.plot(mu, f.max(), 'k+', markersize=15) + plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters + if i % 5 != 0: + plt.yticks([]) + print('%15s: %.3g' % (k, mu)) + f = evolve_csv.with_suffix('.png') # filename + plt.savefig(f, dpi=200) + print(f'Saved {f}') + + +def plot_results(file='path/to/results.csv', dir=''): # Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv') save_dir = Path(file).parent if file else Path(dir) fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True) From 63c19515816972bd932f4e4358a8db1da5c1188e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:17:06 +0200 Subject: [PATCH 06/12] Update --- train.py | 4 ++-- utils/general.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 8b946c2e2954..83a8b6f3aecd 100644 --- a/train.py +++ b/train.py @@ -539,7 +539,7 @@ def main(opt): hyp['anchors'] = 3 opt.noval, opt.nosave, save_dir = True, True, Path(opt.save_dir) # only val/save final epoch # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices - evolve_yaml, evolve_csv = save_dir / 'hyp_evolved.yaml', save_dir / 'evolve.csv' + evolve_yaml, evolve_csv = save_dir / 'hyp_evolve.yaml', save_dir / 'evolve.csv' if opt.bucket: os.system(f'gsutil cp gs://{opt.bucket}/evolve.csv {save_dir}') # download evolve.csv if exists @@ -585,7 +585,7 @@ def main(opt): plot_evolve(evolve_csv) print(f'Hyperparameter evolution complete.\n' f'Best results saved as: {evolve_yaml}\n' - f'Command to train a new model with these hyperparameters: $ python train.py --hyp {evolve_yaml}') + f'Train a new model with these hyperparameters: $ python train.py --hyp {evolve_yaml}') def run(**kwargs): diff --git a/utils/general.py b/utils/general.py index b1936841bcf6..f86c5c187da4 100755 --- a/utils/general.py +++ b/utils/general.py @@ -636,7 +636,7 @@ def print_mutation(results, hyp, save_dir, bucket): # Print to screen print(colorstr('evolve: ') + ', '.join(f'{x.strip():>20s}' for x in keys)) - print(colorstr('evolve: ') + ', '.join(f'{x:20.5g}' for x in vals)) + print(colorstr('evolve: ') + ', '.join(f'{x:20.5g}' for x in vals), end='\n') # Save yaml with open(evolve_yaml, 'w') as f: From 798243989a5a9e5c260875b4d49c2316f77c9178 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:18:17 +0200 Subject: [PATCH 07/12] reset args --- train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/train.py b/train.py index 83a8b6f3aecd..69c01cfc548c 100644 --- a/train.py +++ b/train.py @@ -423,7 +423,7 @@ def parse_opt(known=False): parser.add_argument('--hyp', type=str, default='data/hyps/hyp.scratch.yaml', help='hyperparameters path') parser.add_argument('--epochs', type=int, default=2) parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') - parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=128, help='train, val image size (pixels)') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') parser.add_argument('--rect', action='store_true', help='rectangular training') parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') @@ -453,7 +453,6 @@ def parse_opt(known=False): parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') parser.add_argument('--freeze', type=int, default=0, help='Number of layers to freeze. backbone=10, all=24') opt = parser.parse_known_args()[0] if known else parser.parse_args() - opt.evolve = 2 return opt From 456d24a0ea424b71acc632da6c2e43160683aa1f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:19:38 +0200 Subject: [PATCH 08/12] reset args --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 69c01cfc548c..0c638b16cc81 100644 --- a/train.py +++ b/train.py @@ -421,7 +421,7 @@ def parse_opt(known=False): parser.add_argument('--cfg', type=str, default='', help='model.yaml path') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='dataset.yaml path') parser.add_argument('--hyp', type=str, default='data/hyps/hyp.scratch.yaml', help='hyperparameters path') - parser.add_argument('--epochs', type=int, default=2) + parser.add_argument('--epochs', type=int, default=300) parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') parser.add_argument('--rect', action='store_true', help='rectangular training') @@ -438,7 +438,7 @@ def parse_opt(known=False): parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') - parser.add_argument('--workers', type=int, default=0, help='maximum number of dataloader workers') + parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') parser.add_argument('--project', default='runs/train', help='save to project/name') parser.add_argument('--entity', default=None, help='W&B entity') parser.add_argument('--name', default='exp', help='save to project/name') From b33ccff9c60115019f47eeaa581c62a66df74160 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:25:54 +0200 Subject: [PATCH 09/12] reset args --- train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 0c638b16cc81..a7d61c8c5411 100644 --- a/train.py +++ b/train.py @@ -582,9 +582,9 @@ def main(opt): # Plot results plot_evolve(evolve_csv) - print(f'Hyperparameter evolution complete.\n' - f'Best results saved as: {evolve_yaml}\n' - f'Train a new model with these hyperparameters: $ python train.py --hyp {evolve_yaml}') + print(f'Hyperparameter evolution finished\n' + f"Results saved to {colorstr('bold', save_dir)}" + f'Use best hyperparameters example: $ python train.py --hyp {evolve_yaml}') def run(**kwargs): From 4c48819601ab3154ddf34decc1dd899ae9bd8560 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:41:24 +0200 Subject: [PATCH 10/12] plot_results() fix --- utils/loggers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py index 5315fb73b0c5..d40c0c350fde 100644 --- a/utils/loggers/__init__.py +++ b/utils/loggers/__init__.py @@ -122,7 +122,7 @@ def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): def on_train_end(self, last, best, plots, epoch): # Callback runs on training end if plots: - plot_results(dir=self.save_dir) # save results.png + plot_results(file=self.save_dir / 'results.csv') # save results.png files = ['results.png', 'confusion_matrix.png', *[f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R')]] files = [(self.save_dir / f) for f in files if (self.save_dir / f).exists()] # filter From bc00c9ff1eacc50489de87aa74c19db050cc4626 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:46:18 +0200 Subject: [PATCH 11/12] Cleanup --- .dockerignore | 2 +- .gitignore | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9c9663f006ca..4248cb098cf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,7 @@ coco storage.googleapis.com data/samples/* -**/results*.txt +**/results*.csv *.jpg # Neural Network weights ----------------------------------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index b07134d097dd..e5d02af960af 100755 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ data/* !data/images/bus.jpg !data/*.sh -results*.txt results*.csv # Datasets ------------------------------------------------------------------------------------------------------------- From 5ce1d2b28b1b7ad6783afa815670e9dacf60897f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 4 Aug 2021 16:57:02 +0200 Subject: [PATCH 12/12] Cleanup2 --- utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/general.py b/utils/general.py index f86c5c187da4..15111b727f33 100755 --- a/utils/general.py +++ b/utils/general.py @@ -636,7 +636,7 @@ def print_mutation(results, hyp, save_dir, bucket): # Print to screen print(colorstr('evolve: ') + ', '.join(f'{x.strip():>20s}' for x in keys)) - print(colorstr('evolve: ') + ', '.join(f'{x:20.5g}' for x in vals), end='\n') + print(colorstr('evolve: ') + ', '.join(f'{x:20.5g}' for x in vals), end='\n\n\n') # Save yaml with open(evolve_yaml, 'w') as f: