From b7f613ba6da32941bc86e9188629b572b10db4ad Mon Sep 17 00:00:00 2001
From: Peter Yu <2057325+yukw777@users.noreply.github.com>
Date: Tue, 28 Jul 2020 16:33:28 -0400
Subject: [PATCH] Correct CWD for ddp subprocesses when using Hydra (#2719)

* when hydra is enabled, set the cwd of subprocesses to the original cwd for ddp

* move imports up

* clean up imports
---
 pytorch_lightning/trainer/distrib_data_parallel.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pytorch_lightning/trainer/distrib_data_parallel.py b/pytorch_lightning/trainer/distrib_data_parallel.py
index 2546ec7bb3c7d..7b2c504488bd9 100644
--- a/pytorch_lightning/trainer/distrib_data_parallel.py
+++ b/pytorch_lightning/trainer/distrib_data_parallel.py
@@ -162,7 +162,8 @@ def train_fx(trial_hparams, cluster_manager, _):
 
 
 try:
-    from hydra.utils import to_absolute_path
+    from hydra.utils import to_absolute_path, get_original_cwd
+    from hydra.core.hydra_config import HydraConfig
 except ImportError:
     HYDRA_AVAILABLE = False
 else:
@@ -464,7 +465,12 @@ def spawn_ddp_children(self, model):
             env_copy['LOCAL_RANK'] = f'{local_rank}'
 
             # start process
-            proc = subprocess.Popen(command, env=env_copy)
+            # if hydra is available and initialized, make sure to set the cwd correctly
+            cwd: Optional[str] = None
+            if HYDRA_AVAILABLE:
+                if HydraConfig.initialized():
+                    cwd = get_original_cwd()
+            proc = subprocess.Popen(command, env=env_copy, cwd=cwd)
             self.interactive_ddp_procs.append(proc)
 
             # starting all processes at once can cause issues