From 7fbfd0a2d1b4f84a1beacf306753df7a1030e0c5 Mon Sep 17 00:00:00 2001 From: Jirka Date: Sat, 27 Jun 2020 21:48:03 +0200 Subject: [PATCH] tests: add default_root_dir=tmpdir --- tests/callbacks/test_callbacks.py | 1 + tests/callbacks/test_progress_bar.py | 6 ++++-- tests/loggers/test_all.py | 1 + tests/loggers/test_base.py | 2 +- tests/loggers/test_tensorboard.py | 5 ++++- tests/models/test_amp.py | 16 ++++++--------- tests/models/test_cpu.py | 3 +++ tests/models/test_gpu.py | 20 ++++++++----------- tests/models/test_grad_norm.py | 1 + tests/models/test_hooks.py | 3 ++- tests/models/test_horovod.py | 6 ++---- tests/models/test_restore.py | 12 ++++++----- tests/trainer/test_dataloaders.py | 30 ++++++++++++++-------------- tests/trainer/test_trainer.py | 30 +++++++++++++++++----------- tests/trainer/test_trainer_tricks.py | 6 +++--- 15 files changed, 76 insertions(+), 66 deletions(-) diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py index 93794b56598ce..e6252d8aa7b0e 100644 --- a/tests/callbacks/test_callbacks.py +++ b/tests/callbacks/test_callbacks.py @@ -165,6 +165,7 @@ def on_test_end(self, trainer, pl_module): limit_val_batches=0.1, limit_train_batches=0.2, progress_bar_refresh_rate=0, + default_root_dir=tmpdir, ) assert not test_callback.setup_called diff --git a/tests/callbacks/test_progress_bar.py b/tests/callbacks/test_progress_bar.py index a63fc62585c45..cbb3c7de208bf 100644 --- a/tests/callbacks/test_progress_bar.py +++ b/tests/callbacks/test_progress_bar.py @@ -54,7 +54,7 @@ def test_progress_bar_misconfiguration(): Trainer(callbacks=callbacks) -def test_progress_bar_totals(): +def test_progress_bar_totals(tmpdir): """Test that the progress finishes with the correct total steps processed.""" model = EvalModelTemplate() @@ -63,6 +63,7 @@ def test_progress_bar_totals(): progress_bar_refresh_rate=1, limit_val_batches=1.0, max_epochs=1, + default_root_dir=tmpdir, ) bar = trainer.progress_bar_callback assert 0 == bar.total_train_batches @@ -136,7 +137,7 @@ def test_progress_bar_fast_dev_run(): @pytest.mark.parametrize('refresh_rate', [0, 1, 50]) -def test_progress_bar_progress_refresh(refresh_rate): +def test_progress_bar_progress_refresh(tmpdir, refresh_rate): """Test that the three progress bars get correctly updated when using different refresh rates.""" model = EvalModelTemplate() @@ -177,6 +178,7 @@ def on_test_batch_end(self, trainer, pl_module): limit_train_batches=1.0, num_sanity_val_steps=2, max_epochs=3, + default_root_dir=tmpdir, ) assert trainer.progress_bar_callback.refresh_rate == refresh_rate diff --git a/tests/loggers/test_all.py b/tests/loggers/test_all.py index ca309f42afeee..235dcb9615625 100644 --- a/tests/loggers/test_all.py +++ b/tests/loggers/test_all.py @@ -54,6 +54,7 @@ def log_metrics(self, metrics, step): limit_train_batches=0.2, limit_val_batches=0.5, fast_dev_run=True, + default_root_dir=tmpdir, ) trainer.fit(model) diff --git a/tests/loggers/test_base.py b/tests/loggers/test_base.py index e330e50e581fe..7ca398f16f31d 100644 --- a/tests/loggers/test_base.py +++ b/tests/loggers/test_base.py @@ -102,7 +102,7 @@ def test_multiple_loggers(tmpdir): assert logger2.finalized_status == "success" -def test_multiple_loggers_pickle(tmpdir): +def test_multiple_loggers_pickle(): """Verify that pickling trainer with multiple loggers works.""" logger1 = CustomLogger() diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index e6df2bbc1c691..bf68252f0354d 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -17,7 +17,10 @@ def test_tensorboard_hparams_reload(tmpdir): model = EvalModelTemplate() - trainer = Trainer(max_epochs=1, default_root_dir=tmpdir) + trainer = Trainer( + max_epochs=1, + default_root_dir=tmpdir, + ) trainer.fit(model) folder_path = trainer.logger.log_dir diff --git a/tests/models/test_amp.py b/tests/models/test_amp.py index d8299b89df322..f9c5fe4eab5d0 100644 --- a/tests/models/test_amp.py +++ b/tests/models/test_amp.py @@ -21,7 +21,7 @@ def test_amp_single_gpu(tmpdir, backend): max_epochs=1, gpus=1, distributed_backend=backend, - precision=16 + precision=16, ) model = EvalModelTemplate() @@ -39,8 +39,8 @@ def test_amp_multi_gpu(tmpdir, backend): tutils.set_random_master_port() model = EvalModelTemplate() - - trainer_options = dict( + # tutils.run_model_test(trainer_options, model) + trainer = Trainer( default_root_dir=tmpdir, max_epochs=1, # gpus=2, @@ -48,9 +48,6 @@ def test_amp_multi_gpu(tmpdir, backend): distributed_backend=backend, precision=16, ) - - # tutils.run_model_test(trainer_options, model) - trainer = Trainer(**trainer_options) result = trainer.fit(model) assert result @@ -66,17 +63,15 @@ def test_multi_gpu_wandb(tmpdir, backend): model = EvalModelTemplate() logger = WandbLogger(name='utest') - trainer_options = dict( + # tutils.run_model_test(trainer_options, model) + trainer = Trainer( default_root_dir=tmpdir, max_epochs=1, gpus=2, distributed_backend=backend, precision=16, logger=logger, - ) - # tutils.run_model_test(trainer_options, model) - trainer = Trainer(**trainer_options) result = trainer.fit(model) assert result trainer.test(model) @@ -106,6 +101,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir): precision=16, checkpoint_callback=checkpoint, logger=logger, + default_root_dir=tmpdir, ) trainer.is_slurm_managing_tasks = True result = trainer.fit(model) diff --git a/tests/models/test_cpu.py b/tests/models/test_cpu.py index 808163014adb3..db1dfab63eee8 100644 --- a/tests/models/test_cpu.py +++ b/tests/models/test_cpu.py @@ -29,6 +29,7 @@ def test_cpu_slurm_save_load(tmpdir): limit_train_batches=0.2, limit_val_batches=0.2, checkpoint_callback=ModelCheckpoint(tmpdir), + default_root_dir=tmpdir, ) result = trainer.fit(model) real_global_step = trainer.global_step @@ -64,6 +65,7 @@ def test_cpu_slurm_save_load(tmpdir): max_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir), + default_root_dir=tmpdir, ) model = EvalModelTemplate(**hparams) @@ -220,6 +222,7 @@ def test_running_test_no_val(tmpdir): checkpoint_callback=checkpoint, logger=logger, early_stop_callback=False, + default_root_dir=tmpdir, ) result = trainer.fit(model) diff --git a/tests/models/test_gpu.py b/tests/models/test_gpu.py index 382866b625897..9780f15c005ec 100644 --- a/tests/models/test_gpu.py +++ b/tests/models/test_gpu.py @@ -38,7 +38,9 @@ def test_multi_gpu_model(tmpdir, backend): """Make sure DDP works.""" tutils.set_random_master_port() - trainer_options = dict( + model = EvalModelTemplate() + # tutils.run_model_test(trainer_options, model) + trainer = Trainer( default_root_dir=tmpdir, max_epochs=1, limit_train_batches=0.4, @@ -46,10 +48,6 @@ def test_multi_gpu_model(tmpdir, backend): gpus=[0, 1], distributed_backend=backend, ) - - model = EvalModelTemplate() - # tutils.run_model_test(trainer_options, model) - trainer = Trainer(**trainer_options) result = trainer.fit(model) assert result @@ -63,7 +61,11 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir): """Make sure DDP works with dataloaders passed to fit()""" tutils.set_random_master_port() - trainer_options = dict( + model = EvalModelTemplate() + fit_options = dict(train_dataloader=model.train_dataloader(), + val_dataloaders=model.val_dataloader()) + + trainer = Trainer( default_root_dir=tmpdir, progress_bar_refresh_rate=0, max_epochs=1, @@ -72,12 +74,6 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir): gpus=[0, 1], distributed_backend='ddp' ) - - model = EvalModelTemplate() - fit_options = dict(train_dataloader=model.train_dataloader(), - val_dataloaders=model.val_dataloader()) - - trainer = Trainer(**trainer_options) result = trainer.fit(model, **fit_options) assert result == 1, "DDP doesn't work with dataloaders passed to fit()." diff --git a/tests/models/test_grad_norm.py b/tests/models/test_grad_norm.py index 3c0cd9d6c5b97..233b6ef12cf13 100644 --- a/tests/models/test_grad_norm.py +++ b/tests/models/test_grad_norm.py @@ -88,6 +88,7 @@ def test_grad_tracking(tmpdir, norm_type, rtol=5e-3): logger=logger, track_grad_norm=norm_type, row_log_interval=1, # request grad_norms every batch + default_root_dir=tmpdir, ) result = trainer.fit(model) diff --git a/tests/models/test_hooks.py b/tests/models/test_hooks.py index 78efbd35ff4da..7eaa3c065c04c 100644 --- a/tests/models/test_hooks.py +++ b/tests/models/test_hooks.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize('max_steps', [1, 2, 3]) -def test_on_before_zero_grad_called(max_steps): +def test_on_before_zero_grad_called(tmpdir, max_steps): class CurrentTestModel(EvalModelTemplate): on_before_zero_grad_called = 0 @@ -21,6 +21,7 @@ def on_before_zero_grad(self, optimizer): trainer = Trainer( max_steps=max_steps, num_sanity_val_steps=5, + default_root_dir=tmpdir, ) assert 0 == model.on_before_zero_grad_called trainer.fit(model) diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py index 5f659ade57a38..73e22c754d09f 100644 --- a/tests/models/test_horovod.py +++ b/tests/models/test_horovod.py @@ -146,7 +146,8 @@ def validation_step(self, batch, *args, **kwargs): def test_horovod_multi_optimizer(tmpdir): model = TestGAN(**EvalModelTemplate.get_default_hparams()) - trainer_options = dict( + # fit model + trainer = Trainer( default_root_dir=str(tmpdir), progress_bar_refresh_rate=0, max_epochs=1, @@ -155,9 +156,6 @@ def test_horovod_multi_optimizer(tmpdir): deterministic=True, distributed_backend='horovod', ) - - # fit model - trainer = Trainer(**trainer_options) result = trainer.fit(model) assert result == 1, 'model failed to complete' diff --git a/tests/models/test_restore.py b/tests/models/test_restore.py index c77f7a841f3c3..94102084d37c9 100644 --- a/tests/models/test_restore.py +++ b/tests/models/test_restore.py @@ -38,6 +38,7 @@ def test_running_test_pretrained_model_distrib(tmpdir, backend): logger=logger, gpus=[0, 1], distributed_backend=backend, + default_root_dir=tmpdir, ) # fit model @@ -84,6 +85,7 @@ def test_running_test_pretrained_model_cpu(tmpdir): limit_val_batches=0.2, checkpoint_callback=checkpoint, logger=logger, + default_root_dir=tmpdir, ) # fit model @@ -154,6 +156,7 @@ def test_dp_resume(tmpdir): max_epochs=1, gpus=2, distributed_backend='dp', + default_root_dir=tmpdir, ) # get logger @@ -224,14 +227,13 @@ def test_model_saving_loading(tmpdir): # logger file to get meta logger = tutils.get_default_logger(tmpdir) - trainer_options = dict( + # fit model + trainer = Trainer( max_epochs=1, logger=logger, - checkpoint_callback=ModelCheckpoint(tmpdir) + checkpoint_callback=ModelCheckpoint(tmpdir), + default_root_dir=tmpdir, ) - - # fit model - trainer = Trainer(**trainer_options) result = trainer.fit(model) # traning complete diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 3b44aa69c7164..6020b8bb17435 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -199,7 +199,7 @@ def test_all_dataloaders_passed_to_fit(tmpdir, ckpt_path): default_root_dir=tmpdir, max_epochs=1, limit_val_batches=0.1, - limit_train_batches=0.2 + limit_train_batches=0.2, ) fit_options = dict(train_dataloader=model.dataloader(train=True), val_dataloaders=model.dataloader(train=False)) @@ -232,7 +232,7 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir, ckpt_path): default_root_dir=tmpdir, max_epochs=1, limit_val_batches=0.1, - limit_train_batches=0.2 + limit_train_batches=0.2, ) fit_options = dict(train_dataloader=model.dataloader(train=True), val_dataloaders=[model.dataloader(train=False), @@ -336,7 +336,7 @@ def test_mixing_of_dataloader_options(tmpdir, ckpt_path): default_root_dir=tmpdir, max_epochs=1, limit_val_batches=0.1, - limit_train_batches=0.2 + limit_train_batches=0.2, ) # fit model @@ -401,7 +401,7 @@ def test_inf_train_dataloader(tmpdir, check_interval): trainer = Trainer( default_root_dir=tmpdir, max_epochs=1, - val_check_interval=check_interval + val_check_interval=check_interval, ) result = trainer.fit(model) # verify training completed @@ -440,7 +440,7 @@ def test_error_on_zero_len_dataloader(tmpdir): max_epochs=1, limit_train_batches=0.1, limit_val_batches=0.1, - limit_test_batches=0.1 + limit_test_batches=0.1, ) trainer.fit(model) @@ -453,13 +453,6 @@ def test_warning_with_few_workers(tmpdir, ckpt_path): model = EvalModelTemplate() # logger file to get meta - trainer_options = dict( - default_root_dir=tmpdir, - max_epochs=1, - limit_val_batches=0.1, - limit_train_batches=0.2 - ) - train_dl = model.dataloader(train=True) train_dl.num_workers = 0 @@ -471,7 +464,12 @@ def test_warning_with_few_workers(tmpdir, ckpt_path): fit_options = dict(train_dataloader=train_dl, val_dataloaders=val_dl) - trainer = Trainer(**trainer_options) + trainer = Trainer( + default_root_dir=tmpdir, + max_epochs=1, + limit_val_batches=0.1, + limit_train_batches=0.2, + ) # fit model with pytest.warns(UserWarning, match='train'): @@ -488,7 +486,7 @@ def test_warning_with_few_workers(tmpdir, ckpt_path): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason='Test requires multiple GPUs') -def test_dataloader_reinit_for_subclass(): +def test_dataloader_reinit_for_subclass(tmpdir): class CustomDataLoader(torch.utils.data.DataLoader): def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, @@ -505,6 +503,7 @@ def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, gpus=[0, 1], num_nodes=1, distributed_backend='ddp', + default_root_dir=tmpdir, ) class CustomDummyObj: @@ -534,7 +533,7 @@ class CustomSampler(torch.utils.data.Sampler): @pytest.mark.skipif(torch.cuda.device_count() < 3, reason='Test requires multiple GPUs') -def test_batch_size_smaller_than_num_gpus(): +def test_batch_size_smaller_than_num_gpus(tmpdir): # we need at least 3 gpus for this test num_gpus = 3 batch_size = 3 @@ -576,6 +575,7 @@ def train_dataloader(self): limit_train_batches=0.1, limit_val_batches=0, gpus=num_gpus, + default_root_dir=tmpdir, ) # we expect the reduction for the metrics also to happen on the last batch diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 1a2b3294894f6..ca9ac4c274844 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -38,7 +38,8 @@ def test_no_val_module(monkeypatch, tmpdir, tmpdir_server, url_ckpt): trainer = Trainer( max_epochs=1, logger=logger, - checkpoint_callback=ModelCheckpoint(tmpdir) + checkpoint_callback=ModelCheckpoint(tmpdir), + default_root_dir=tmpdir, ) # fit model result = trainer.fit(model) @@ -79,7 +80,8 @@ def test_no_val_end_module(monkeypatch, tmpdir, tmpdir_server, url_ckpt): trainer = Trainer( max_epochs=1, logger=logger, - checkpoint_callback=ModelCheckpoint(tmpdir) + checkpoint_callback=ModelCheckpoint(tmpdir), + default_root_dir=tmpdir, ) result = trainer.fit(model) @@ -166,11 +168,13 @@ def _optimizer_step(self, epoch, batch_idx, optimizer, model = EvalModelTemplate() schedule = {1: 2, 3: 4} - trainer = Trainer(accumulate_grad_batches=schedule, - limit_train_batches=0.1, - limit_val_batches=0.1, - max_epochs=2, - default_root_dir=tmpdir) + trainer = Trainer( + accumulate_grad_batches=schedule, + limit_train_batches=0.1, + limit_val_batches=0.1, + max_epochs=2, + default_root_dir=tmpdir, + ) # for the test trainer.optimizer_step = _optimizer_step @@ -298,7 +302,8 @@ def test_model_checkpoint_only_weights(tmpdir): trainer = Trainer( max_epochs=1, - checkpoint_callback=ModelCheckpoint(tmpdir, save_weights_only=True) + checkpoint_callback=ModelCheckpoint(tmpdir, save_weights_only=True), + default_root_dir=tmpdir, ) # fit model result = trainer.fit(model) @@ -432,7 +437,7 @@ def test_trainer_max_steps_and_epochs(tmpdir): trainer_options.update( default_root_dir=tmpdir, max_epochs=3, - max_steps=num_train_samples + 10 + max_steps=num_train_samples + 10, ) # fit model @@ -446,7 +451,7 @@ def test_trainer_max_steps_and_epochs(tmpdir): # define less train epochs than steps trainer_options.update( max_epochs=2, - max_steps=trainer_options['max_epochs'] * 2 * num_train_samples + max_steps=trainer_options['max_epochs'] * 2 * num_train_samples, ) # fit model @@ -469,7 +474,7 @@ def test_trainer_min_steps_and_epochs(tmpdir): early_stop_callback=EarlyStopping(monitor='val_loss', min_delta=1.0), val_check_interval=2, min_epochs=1, - max_epochs=2 + max_epochs=2, ) # define less min steps than 1 epoch @@ -592,7 +597,7 @@ def load_from_checkpoint(cls, checkpoint_path, *args, **kwargs): assert loaded_checkpoint_path == ckpt_path -def test_disabled_validation(): +def test_disabled_validation(tmpdir): """Verify that `limit_val_batches=0` disables the validation loop unless `fast_dev_run=True`.""" class CurrentModel(EvalModelTemplate): @@ -617,6 +622,7 @@ def validation_epoch_end(self, *args, **kwargs): limit_train_batches=0.4, limit_val_batches=0.0, fast_dev_run=False, + default_root_dir=tmpdir, ) trainer = Trainer(**trainer_options) diff --git a/tests/trainer/test_trainer_tricks.py b/tests/trainer/test_trainer_tricks.py index 32415ffe2e9ac..56c1416ae2e4a 100755 --- a/tests/trainer/test_trainer_tricks.py +++ b/tests/trainer/test_trainer_tricks.py @@ -118,7 +118,7 @@ def test_model_reset_correctly(tmpdir): # logger file to get meta trainer = Trainer( default_root_dir=tmpdir, - max_epochs=1 + max_epochs=1, ) before_state_dict = model.state_dict() @@ -141,7 +141,7 @@ def test_trainer_reset_correctly(tmpdir): # logger file to get meta trainer = Trainer( default_root_dir=tmpdir, - max_epochs=1 + max_epochs=1, ) changed_attributes = ['max_steps', @@ -224,7 +224,7 @@ def test_error_on_dataloader_passed_to_fit(tmpdir): max_epochs=1, limit_val_batches=0.1, limit_train_batches=0.2, - auto_scale_batch_size='power' + auto_scale_batch_size='power', ) fit_options = dict(train_dataloader=model.dataloader(train=True))