[WB-4478] Feature/tb sync (#1852)

tests/fixtures/wandb/offline-run-20210216_154407-g9dvvkua/files/code/standalone_tests/code-toad.py

@@ -0,0 +1,8 @@
+import wandb
+import os
+
+os.environ["WANDB_CODE_DIR"] = "."
+
+wandb.init(project="code-toad")
+
+# wandb.run.log_code()

tests/fixtures/wandb/offline-run-20210216_154407-g9dvvkua/files/config.yaml

@@ -0,0 +1,13 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.20.dev1
+    code_path: code/standalone_tests/code-toad.py
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.7.5
+    t:
+      4: 3.7.5
+      5: 0.10.20.dev1

tests/fixtures/wandb/offline-run-20210216_154407-g9dvvkua/files/diff.patch

@@ -0,0 +1,146 @@
+diff --git a/wandb/jupyter.py b/wandb/jupyter.py
+index cb611260..2512f540 100644
+--- a/wandb/jupyter.py
++++ b/wandb/jupyter.py
+@@ -267,7 +267,7 @@ class Notebook(object):
+             if os.path.exists(relpath):
+                 shutil.copy(
+                     relpath,
+-                    os.path.join(self.settings.code_dir, os.path.basename(relpath)),
++                    os.path.join(self.settings._tmp_code_dir, os.path.basename(relpath)),
+                 )
+                 return True
+
+@@ -276,7 +276,7 @@ class Notebook(object):
+         if colab_ipynb:
+             with open(
+                 os.path.join(
+-                    self.settings.code_dir, colab_ipynb["metadata"]["colab"]["name"]
++                    self.settings._tmp_code_dir, colab_ipynb["metadata"]["colab"]["name"]
+                 ),
+                 "w",
+                 encoding="utf-8",
+@@ -287,7 +287,7 @@ class Notebook(object):
+         kaggle_ipynb = attempt_kaggle_load_ipynb()
+         if kaggle_ipynb and len(kaggle_ipynb["cells"]) > 0:
+             with open(
+-                os.path.join(self.settings.code_dir, kaggle_ipynb["metadata"]["name"]),
++                os.path.join(self.settings._tmp_code_dir, kaggle_ipynb["metadata"]["name"]),
+                 "w",
+                 encoding="utf-8",
+             ) as f:
+@@ -355,7 +355,7 @@ class Notebook(object):
+             wandb.run.config.persist()
+             wandb.util.mkdir_exists_ok(os.path.join(wandb.run.dir, "code"))
+             with open(
+-                os.path.join(self.settings.code_dir, "_session_history.ipynb"),
++                os.path.join(self.settings._tmp_code_dir, "_session_history.ipynb"),
+                 "w",
+                 encoding="utf-8",
+             ) as f:
+diff --git a/wandb/sdk/wandb_init.py b/wandb/sdk/wandb_init.py
+index 1bbc5f30..78a4c0f3 100644
+--- a/wandb/sdk/wandb_init.py
++++ b/wandb/sdk/wandb_init.py
+@@ -298,7 +298,7 @@ class _WandbInit(object):
+         filesystem._safe_makedirs(os.path.dirname(settings.log_internal))
+         filesystem._safe_makedirs(os.path.dirname(settings.sync_file))
+         filesystem._safe_makedirs(settings.files_dir)
+-        filesystem._safe_makedirs(settings.code_dir)
++        filesystem._safe_makedirs(settings._tmp_code_dir)
+
+         if settings.symlink:
+             self._safe_symlink(
+diff --git a/wandb/sdk/wandb_run.py b/wandb/sdk/wandb_run.py
+index 60781e68..e28bf251 100644
+--- a/wandb/sdk/wandb_run.py
++++ b/wandb/sdk/wandb_run.py
+@@ -652,10 +652,10 @@ class Run(object):
+                 save_name = os.path.relpath(file_path, root)
+                 art.add_file(file_path, name=save_name)
+         # Add any manually staged files such is ipynb notebooks
+-        for dirpath, _, files in os.walk(self._settings.code_dir):
++        for dirpath, _, files in os.walk(self._settings._tmp_code_dir):
+             for fname in files:
+                 file_path = os.path.join(dirpath, fname)
+-                save_name = os.path.relpath(file_path, self._settings.code_dir)
++                save_name = os.path.relpath(file_path, self._settings._tmp_code_dir)
+                 files_added = True
+                 art.add_file(file_path, name=save_name)
+         if not files_added:
+@@ -1497,16 +1497,16 @@ class Run(object):
+
+     def _on_start(self) -> None:
+         # TODO: make offline mode in jupyter use HTML
+-        if self._settings._offline:
+-            wandb.termlog("Offline run mode, not syncing to the cloud.")
+-
+         if self._settings._offline:
+             wandb.termlog(
+                 (
+                     "W&B syncing is set to `offline` in this directory.  "
+-                    "Run `wandb online` to enable cloud syncing."
++                    "Run `wandb online` or set WANDB_MODE=online to enable cloud syncing."
+                 )
+             )
++        print("GOT SETTINGS", self._settings.code_dir)
++        if self._settings.code_dir is not None:
++            self.log_code(self._settings.code_dir)
+         if self._run_obj and not self._settings._silent:
+             self._display_run()
+         if self._backend and not self._settings._offline:
+diff --git a/wandb/sdk/wandb_settings.py b/wandb/sdk/wandb_settings.py
+index 58c529d5..9995db47 100644
+--- a/wandb/sdk/wandb_settings.py
++++ b/wandb/sdk/wandb_settings.py
+@@ -85,6 +85,7 @@ env_settings: Dict[str, Optional[str]] = dict(
+     host=None,
+     username=None,
+     disable_code=None,
++    code_dir=None,
+     anonymous=None,
+     ignore_globs=None,
+     resume=None,
+@@ -216,7 +217,6 @@ class Settings(object):
+     sync_dir_spec: Optional[str] = None
+     files_dir_spec: Optional[str] = None
+     tmp_dir_spec: Optional[str] = None
+-    code_dir_spec: Optional[str] = None
+     log_symlink_user_spec: Optional[str] = None
+     log_symlink_internal_spec: Optional[str] = None
+     sync_symlink_latest_spec: Optional[str] = None
+@@ -228,6 +228,7 @@ class Settings(object):
+     show_errors: bool = True
+     email: Optional[str] = None
+     save_code: Optional[bool] = None
++    code_dir: Optional[str] = None
+     program_relpath: Optional[str] = None
+     host: Optional[str]
+
+@@ -321,7 +322,6 @@ class Settings(object):
+         resume_fname_spec="{wandb_dir}/wandb-resume.json",
+         files_dir_spec="{wandb_dir}/{run_mode}-{timespec}-{run_id}/files",
+         tmp_dir_spec="{wandb_dir}/{run_mode}-{timespec}-{run_id}/tmp",
+-        code_dir_spec="{wandb_dir}/{run_mode}-{timespec}-{run_id}/tmp/code",
+         symlink=None,  # probed
+         # where files are temporary stored when saving
+         # files_dir=None,
+@@ -336,6 +336,7 @@ class Settings(object):
+         disable_code=None,
+         ignore_globs=None,
+         save_code=None,
++        code_dir=None,
+         program_relpath=None,
+         git_remote=None,
+         dev_prod=None,  # in old settings files, TODO: support?
+@@ -516,8 +517,8 @@ class Settings(object):
+         return self._path_convert(self.tmp_dir_spec)
+
+     @property
+-    def code_dir(self) -> str:
+-        return self._path_convert(self.code_dir_spec)
++    def _tmp_code_dir(self) -> str:
++        return os.path.join(self.tmp_dir, "code")
+
+     @property
+     def log_symlink_user(self) -> str: