diff --git a/.run_local_tests.sh b/.run_local_tests.sh index b29fbbf8a8533..5e015941c2976 100644 --- a/.run_local_tests.sh +++ b/.run_local_tests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # use this to run tests rm -rf _ckpt_* rm -rf tests/save_dir* diff --git a/pytorch_lightning/callbacks/pt_callbacks.py b/pytorch_lightning/callbacks/pt_callbacks.py index 1c54f98a9fb9f..12eae509097aa 100644 --- a/pytorch_lightning/callbacks/pt_callbacks.py +++ b/pytorch_lightning/callbacks/pt_callbacks.py @@ -27,7 +27,7 @@ def set_params(self, params): self.params = params def set_model(self, model): - if type(model) is LightningDistributedDataParallel: + if isinstance(model, LightningDistributedDataParallel): model = model.module self.model = model @@ -43,7 +43,6 @@ def on_epoch_begin(self, epoch, logs=None): on_epoch_begin(epoch=2, logs={'val_loss': 0.2}) """ - pass def on_epoch_end(self, epoch, logs=None): pass @@ -56,7 +55,6 @@ def on_batch_begin(self, batch, logs=None): batch (Tensor): current batch tensor logs (dict): key-value pairs of quantities to monitor """ - pass def on_batch_end(self, batch, logs=None): pass @@ -143,7 +141,7 @@ def check_metrics(self, logs): if monitor_val is None: if self.strict: raise RuntimeError(error_msg) - elif self.verbose > 0: + if self.verbose > 0: warnings.warn(error_msg, RuntimeWarning) return False @@ -399,7 +397,7 @@ def __init__(self, scheduling: dict): if minimal_epoch < 1: msg = f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct" raise IndexError(msg) - elif minimal_epoch != 1: # if user didnt define first epoch accumulation factor + if minimal_epoch != 1: # if user didnt define first epoch accumulation factor scheduling.update({1: 1}) self.scheduling = scheduling diff --git a/pytorch_lightning/core/hooks.py b/pytorch_lightning/core/hooks.py index c89b82e00a101..6c5f74b2f362e 100644 --- a/pytorch_lightning/core/hooks.py +++ b/pytorch_lightning/core/hooks.py @@ -32,14 +32,12 @@ def on_sanity_check_start(self): .. warning:: will be deprecated. :return: """ - pass def on_train_start(self): """Called at the beginning of training before sanity check :return: """ # do something at the start of training - pass def on_train_end(self): """ @@ -47,7 +45,6 @@ def on_train_end(self): :return: """ # do something at the end of training - pass def on_batch_start(self, batch): """Called in the training loop before anything happens for that batch. @@ -56,32 +53,26 @@ def on_batch_start(self, batch): :return: """ # do something when the batch starts - pass def on_batch_end(self): """Called in the training loop after the batch.""" # do something when the batch ends - pass def on_epoch_start(self): """Called in the training loop at the very beginning of the epoch.""" # do something when the epoch starts - pass def on_epoch_end(self): """Called in the training loop at the very end of the epoch.""" # do something when the epoch ends - pass def on_pre_performance_check(self): """Called at the very beginning of the validation loop.""" # do something before validation starts - pass def on_post_performance_check(self): """Called at the very end of the validation loop.""" # do something before validation end - pass def on_before_zero_grad(self, optimizer): """Called after optimizer.step() and before optimizer.zero_grad() @@ -99,7 +90,6 @@ def on_before_zero_grad(self, optimizer): :return: """ # do something with the optimizer or inspect it. - pass def on_after_backward(self): """Called after loss.backward() and before optimizers do anything. @@ -122,7 +112,6 @@ def on_after_backward(self): global_step=self.trainer.global_step) """ - pass def backward(self, use_amp, loss, optimizer, optimizer_idx): """Override backward with your own implementation if you need to diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 9c1da34b1b66f..0ec7d03ff5da2 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -253,7 +253,6 @@ def training_step(self, batch, batch_idx, hiddens): You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to break out of the current training epoch early. """ - pass def validation_step(self, *args, **kwargs): r""" @@ -326,7 +325,6 @@ def validation_step(self, batch, batch_idx, dataset_idx): .. note:: When the validation_step is called, the model has been put in eval mode and PyTorch gradients have been disabled. At the end of validation, model goes back to training mode and gradients are enabled. """ - pass def test_step(self, *args, **kwargs): """return whatever outputs will need to be aggregated in test_end @@ -395,7 +393,6 @@ def test_step(self, batch, batch_idx, dataset_idx): The `dataset_idx` corresponds to the order of datasets returned in `test_dataloader`. """ - pass def validation_end(self, outputs): """Outputs has the appended output after each validation step. @@ -467,7 +464,6 @@ def validation_end(self, outputs): return results """ - pass def test_end(self, outputs): """Outputs has the appended output after each test step. @@ -532,7 +528,6 @@ def test_end(self, outputs): return results """ - pass def configure_ddp(self, model, device_ids): r""" @@ -842,8 +837,7 @@ def tbptt_split_batch(self, batch, split_size): Each returned batch split is passed separately to training_step(...). """ - time_dims = [len(x[0]) for x in batch if isinstance( - x, torch.Tensor) or isinstance(x, collections.Sequence)] + time_dims = [len(x[0]) for x in batch if isinstance(x, (torch.Tensor, collections.Sequence))] assert len(time_dims) >= 1, "Unable to determine batch time dimension" assert all(x == time_dims[0] for x in time_dims), "Batch time dimension length is ambiguous" @@ -1192,7 +1186,6 @@ def on_load_checkpoint(self, checkpoint): .. note:: Lighting auto-restores global step, epoch, and all training state including amp scaling. No need for you to restore anything regarding training. """ - pass def on_save_checkpoint(self, checkpoint): r""" @@ -1216,7 +1209,6 @@ def on_save_checkpoint(self, checkpoint): for you to store anything about training. """ - pass def load_hparams_from_tags_csv(tags_csv): @@ -1236,7 +1228,7 @@ def load_hparams_from_tags_csv(tags_csv): def convert(val): constructors = [int, float, str] - if type(val) is str: + if isinstance(val, str): if val.lower() == 'true': return True if val.lower() == 'false': diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index a1ae908d36782..fbfe79ffb38c0 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -77,7 +77,7 @@ def get_variable_sizes(self): if isinstance(input_, (list, tuple)): # pragma: no cover in_size = [] for x in input_: - if type(x) is list: + if isinstance(x, list): in_size.append(len(x)) else: in_size.append(x.size()) @@ -97,7 +97,6 @@ def get_variable_sizes(self): self.in_sizes = in_sizes self.out_sizes = out_sizes assert len(in_sizes) == len(out_sizes) - return def get_layer_names(self): '''Collect Layer Names''' @@ -112,7 +111,6 @@ def get_layer_names(self): self.layer_names = names self.layer_types = layer_types - return def get_parameter_sizes(self): '''Get sizes of all parameters in `model`''' @@ -120,13 +118,10 @@ def get_parameter_sizes(self): sizes = [] for _, m in mods: p = list(m.parameters()) - modsz = [] - for j in range(len(p)): - modsz.append(np.array(p[j].size())) + modsz = [np.array(param.size()) for param in p] sizes.append(modsz) self.param_sizes = sizes - return def get_parameter_nums(self): '''Get number of parameters in each layer''' @@ -137,7 +132,6 @@ def get_parameter_nums(self): all_params += np.prod(p) param_nums.append(all_params) self.param_nums = param_nums - return def make_summary(self): ''' diff --git a/pytorch_lightning/core/saving.py b/pytorch_lightning/core/saving.py index a9015afc83692..ef44c7a5f15a7 100644 --- a/pytorch_lightning/core/saving.py +++ b/pytorch_lightning/core/saving.py @@ -7,14 +7,12 @@ def on_load_checkpoint(self, checkpoint): :param checkpoint: :return: """ - pass def on_save_checkpoint(self, checkpoint): """ Give the model a chance to add something to the checkpoint. state_dict is already there """ - pass # ------------------------- # OPTIONAL HOOKS @@ -24,11 +22,9 @@ def on_hpc_save(self, checkpoint): Hook to do whatever you need right before Slurm manager saves the model :return: """ - pass def on_hpc_load(self, checkpoint): """ Hook to do whatever you need right before Slurm manager loads the model :return: """ - pass diff --git a/pytorch_lightning/loggers/base.py b/pytorch_lightning/loggers/base.py index 76bd0d2d9a42c..1835bba2bf1d5 100644 --- a/pytorch_lightning/loggers/base.py +++ b/pytorch_lightning/loggers/base.py @@ -43,18 +43,15 @@ def log_hyperparams(self, params): def save(self): """Save log data.""" - pass def finalize(self, status): """Do any processing that is necessary to finalize an experiment. :param status: Status that the experiment finished with (e.g. success, failed, aborted) """ - pass def close(self): """Do any cleanup that is necessary to close an experiment.""" - pass @property def rank(self): diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index 63df5a6443025..fb632b96ca075 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -143,5 +143,5 @@ def _get_next_version(self): if len(existing_versions) == 0: return 0 - else: - return max(existing_versions) + 1 + + return max(existing_versions) + 1 diff --git a/pytorch_lightning/overrides/data_parallel.py b/pytorch_lightning/overrides/data_parallel.py index adf3f631c91cb..5eaadac50d0e8 100644 --- a/pytorch_lightning/overrides/data_parallel.py +++ b/pytorch_lightning/overrides/data_parallel.py @@ -25,7 +25,7 @@ def get_a_var(obj): # pragma: no cover if isinstance(obj, torch.Tensor): return obj - if isinstance(obj, list) or isinstance(obj, tuple): + if isinstance(obj, (list, tuple)): for result in map(get_a_var, obj): if isinstance(result, torch.Tensor): return result @@ -56,10 +56,10 @@ def forward(self, *inputs, **kwargs): # lightning if self.module.training: return self.module.training_step(*inputs[0], **kwargs[0]) - elif self.module.testing: + if self.module.testing: return self.module.test_step(*inputs[0], **kwargs[0]) - else: - return self.module.validation_step(*inputs[0], **kwargs[0]) + + return self.module.validation_step(*inputs[0], **kwargs[0]) replicas = self.replicate(self.module, self.device_ids[:len(inputs)]) outputs = self.parallel_apply(replicas, inputs, kwargs) diff --git a/pytorch_lightning/trainer/distrib_data_parallel.py b/pytorch_lightning/trainer/distrib_data_parallel.py index b66981b995450..96adeaf0afec6 100644 --- a/pytorch_lightning/trainer/distrib_data_parallel.py +++ b/pytorch_lightning/trainer/distrib_data_parallel.py @@ -246,7 +246,7 @@ def set_nvidia_flags(self, is_slurm_managing_tasks, data_parallel_device_ids): # when slurm is managing the task it sets the visible devices if not is_slurm_managing_tasks: - if type(data_parallel_device_ids) is int: + if isinstance(data_parallel_device_ids, int): id_str = ','.join(str(x) for x in list(range(data_parallel_device_ids))) os.environ["CUDA_VISIBLE_DEVICES"] = id_str else: diff --git a/pytorch_lightning/trainer/distrib_parts.py b/pytorch_lightning/trainer/distrib_parts.py index db21e4d036622..d1417d2c5210f 100644 --- a/pytorch_lightning/trainer/distrib_parts.py +++ b/pytorch_lightning/trainer/distrib_parts.py @@ -400,24 +400,24 @@ def transfer_batch_to_gpu(self, batch, gpu_id): if callable(getattr(batch, 'cuda', None)): return batch.cuda(gpu_id) - elif callable(getattr(batch, 'to', None)): + if callable(getattr(batch, 'to', None)): return batch.to(torch.device('cuda', gpu_id)) # when list - elif isinstance(batch, list): + if isinstance(batch, list): for i, x in enumerate(batch): batch[i] = self.transfer_batch_to_gpu(x, gpu_id) return batch # when tuple - elif isinstance(batch, tuple): + if isinstance(batch, tuple): batch = list(batch) for i, x in enumerate(batch): batch[i] = self.transfer_batch_to_gpu(x, gpu_id) return tuple(batch) # when dict - elif isinstance(batch, dict): + if isinstance(batch, dict): for k, v in batch.items(): batch[k] = self.transfer_batch_to_gpu(v, gpu_id) @@ -463,7 +463,7 @@ def dp_train(self, model): # create list of device ids device_ids = self.data_parallel_device_ids - if type(device_ids) is int: + if isinstance(device_ids, int): device_ids = list(range(device_ids)) model = LightningDataParallel(model, device_ids=device_ids) @@ -472,7 +472,7 @@ def dp_train(self, model): def normalize_parse_gpu_string_input(s): - if type(s) is str: + if isinstance(s, str): if s == '-1': return -1 else: @@ -496,10 +496,7 @@ def check_gpus_data_type(gpus): :return: return unmodified gpus variable """ - if (gpus is not None and - type(gpus) is not int and - type(gpus) is not str and - type(gpus) is not list): # noqa E129 + if gpus is not None and type(gpus) not in (int, str, list): raise MisconfigurationException("GPUs must be int, string or list of ints or None.") @@ -507,13 +504,14 @@ def normalize_parse_gpu_input_to_list(gpus): assert gpus is not None if isinstance(gpus, list): return gpus - else: # must be an int - if not gpus: # gpus==0 - return None - elif gpus == -1: - return get_all_available_gpus() - else: - return list(range(gpus)) + + # must be an int + if not gpus: # gpus==0 + return None + if gpus == -1: + return get_all_available_gpus() + + return list(range(gpus)) def sanitize_gpu_ids(gpus): @@ -552,7 +550,7 @@ def parse_gpu_ids(gpus): check_gpus_data_type(gpus) # Handle the case when no gpus are requested - if gpus is None or type(gpus) is int and gpus == 0: + if gpus is None or isinstance(gpus, int) and gpus == 0: return None # We know user requested GPUs therefore if some of the @@ -576,7 +574,7 @@ def determine_root_gpu_device(gpus): return None assert isinstance(gpus, list), "gpus should be a list" - assert len(gpus), "gpus should be a non empty list" + assert len(gpus) > 0, "gpus should be a non empty list" # set root gpu root_gpu = gpus[0] diff --git a/pytorch_lightning/trainer/logging.py b/pytorch_lightning/trainer/logging.py index 9ff5c3db85076..2025bd511eb49 100644 --- a/pytorch_lightning/trainer/logging.py +++ b/pytorch_lightning/trainer/logging.py @@ -65,7 +65,7 @@ def log_metrics(self, metrics, grad_norm_dic, step=None): def add_tqdm_metrics(self, metrics): for k, v in metrics.items(): - if type(v) is torch.Tensor: + if isinstance(v, torch.Tensor): v = v.item() self.tqdm_metrics[k] = v @@ -76,7 +76,7 @@ def metrics_to_scalars(self, metrics): if isinstance(v, torch.Tensor): v = v.item() - if type(v) is dict: + if isinstance(v, dict): v = self.metrics_to_scalars(v) new_metrics[k] = v @@ -148,7 +148,7 @@ def process_output(self, output, train=False): try: loss = output['loss'] except Exception: - if type(output) is torch.Tensor: + if isinstance(output, torch.Tensor): loss = output else: raise RuntimeError( @@ -181,7 +181,7 @@ def reduce_distributed_output(self, output, num_gpus): # when using DP, we get one output per gpu # average outputs and return - if type(output) is torch.Tensor: + if isinstance(output, torch.Tensor): return output.mean() for k, v in output.items(): diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 914ef27246ba1..0b15100606f6c 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -660,7 +660,7 @@ def __set_root_gpu(self, gpus): # set root gpu root_gpu = 0 - if type(gpus) is list: + if isinstance(gpus, list): root_gpu = gpus[0] return root_gpu @@ -670,8 +670,7 @@ def num_gpus(self): gpus = self.data_parallel_device_ids if gpus is None: return 0 - else: - return len(gpus) + return len(gpus) @property def data_parallel(self): @@ -769,13 +768,13 @@ def init_optimizers(self, optimizers): return [optimizers], [] # two lists - elif len(optimizers) == 2 and isinstance(optimizers[0], list): + if len(optimizers) == 2 and isinstance(optimizers[0], list): optimizers, lr_schedulers = optimizers lr_schedulers, self.reduce_lr_on_plateau_scheduler = self.configure_schedulers(lr_schedulers) return optimizers, lr_schedulers # single list or tuple - elif isinstance(optimizers, list) or isinstance(optimizers, tuple): + if isinstance(optimizers, (list, tuple)): return optimizers, [] def configure_schedulers(self, schedulers): diff --git a/tests/test_restore_models.py b/tests/test_restore_models.py index 349e649e94592..57d28040f1016 100644 --- a/tests/test_restore_models.py +++ b/tests/test_restore_models.py @@ -58,9 +58,9 @@ def test_running_test_pretrained_model_ddp(tmpdir): def test_running_test_pretrained_model(tmpdir): + """Verify test() on pretrained model.""" tutils.reset_seed() - """Verify test() on pretrained model""" hparams = tutils.get_hparams() model = LightningTestModel(hparams) @@ -97,9 +97,9 @@ def test_running_test_pretrained_model(tmpdir): def test_load_model_from_checkpoint(tmpdir): + """Verify test() on pretrained model.""" tutils.reset_seed() - """Verify test() on pretrained model""" hparams = tutils.get_hparams() model = LightningTestModel(hparams) @@ -138,9 +138,9 @@ def test_load_model_from_checkpoint(tmpdir): def test_running_test_pretrained_model_dp(tmpdir): + """Verify test() on pretrained model.""" tutils.reset_seed() - """Verify test() on pretrained model""" if not tutils.can_run_gpu_test(): return diff --git a/tests/test_trainer.py b/tests/test_trainer.py index ddc74b40f8ca4..ad2748506fc9f 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -97,11 +97,11 @@ class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase): def test_gradient_accumulation_scheduling(tmpdir): - tutils.reset_seed() - """ Test grad accumulation by the freq of optimizer updates """ + tutils.reset_seed() + # test incorrect configs with pytest.raises(IndexError): assert Trainer(accumulate_grad_batches={0: 3, 1: 4, 4: 6})