From 1eebf54535c0cc069799b86b725af8dfa230c675 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 8 Jun 2020 16:03:32 -0400 Subject: [PATCH 1/3] training batch clean up --- pytorch_lightning/trainer/trainer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index aff19fb253f38..5fa5ff8e45668 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -345,6 +345,7 @@ def __init__( "track_grad_norm can be an int, a float or 'inf' (infinity norm).") self.track_grad_norm = float(track_grad_norm) + import pdb; pdb.set_trace() self.on_gpu = True if (gpus and torch.cuda.is_available()) else False # tpu config From 78dbb7d085483890f94065759a5468fa680447df Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 8 Jun 2020 16:18:03 -0400 Subject: [PATCH 2/3] training batch clean up --- pytorch_lightning/trainer/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 5fa5ff8e45668..3742258f273e2 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -1034,7 +1034,7 @@ def run_pretrain_routine(self, model: LightningModule): self.early_stop_callback._validate_condition_metric(callback_metrics) # clear cache before training - if self.on_gpu: + if self.on_gpu and self.root_gpu is not None: # use context because of: # https://discuss.pytorch.org/t/out-of-memory-when-i-use-torch-cuda-empty-cache/57898 with torch.cuda.device(f'cuda:{self.root_gpu}'): From 2cfc1da3a3fbb0894f5149218942c68ea177c0e7 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 8 Jun 2020 16:18:29 -0400 Subject: [PATCH 3/3] training batch clean up --- pytorch_lightning/trainer/trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 3742258f273e2..1f5b73f9be364 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -345,7 +345,6 @@ def __init__( "track_grad_norm can be an int, a float or 'inf' (infinity norm).") self.track_grad_norm = float(track_grad_norm) - import pdb; pdb.set_trace() self.on_gpu = True if (gpus and torch.cuda.is_available()) else False # tpu config