implement XCLIM_TESTDATA_BRANCH and XCLIM_PREFETCH_TESTING_DATA e…

…nvironment variables, remove obsolete subset-specific testing data, adjust testing data fetching logic

xclim/testing/tests/conftest.py

@@ -22,8 +22,8 @@
 from xclim.testing.utils import _default_cache_dir  # noqa
 from xclim.testing.utils import open_dataset as _open_dataset
 
-MAIN_TESTDATA_BRANCH = os.getenv("MAIN_TESTDATA_BRANCH", "main")
-SKIP_TEST_DATA = os.getenv("SKIP_TEST_DATA")
+TESTDATA_BRANCH = os.getenv("XCLIM_TESTDATA_BRANCH", "main")
+PREFETCH_TESTING_DATA = os.getenv("XCLIM_PREFETCH_TESTING_DATA")
 
 
 @pytest.fixture
@@ -518,7 +518,7 @@ def cmip3_day_tas(threadsafe_data_dir):
     ds = _open_dataset(
         "cmip3/tas.sresb1.giss_model_e_r.run1.atm.da.nc",
         cache_dir=threadsafe_data_dir,
-        branch=MAIN_TESTDATA_BRANCH,
+        branch=TESTDATA_BRANCH,
     )
     yield ds.tas
     ds.close()
@@ -527,7 +527,7 @@ def cmip3_day_tas(threadsafe_data_dir):
 @pytest.fixture(scope="session")
 def open_dataset(threadsafe_data_dir):
     def _open_session_scoped_file(
-        file: str | os.PathLike, branch: str = MAIN_TESTDATA_BRANCH, **xr_kwargs
+        file: str | os.PathLike, branch: str = TESTDATA_BRANCH, **xr_kwargs
     ):
         return _open_dataset(
             file, cache_dir=threadsafe_data_dir, branch=branch, **xr_kwargs
@@ -544,7 +544,7 @@ def add_imports(xdoctest_namespace, threadsafe_data_dir) -> None:
     ns["xr"] = xclim.testing  # xr.open_dataset(...) -> xclim.testing.open_dataset(...)
     ns["xclim"] = xclim
     ns["open_dataset"] = partial(
-        _open_dataset, cache_dir=threadsafe_data_dir, branch=MAIN_TESTDATA_BRANCH
+        _open_dataset, cache_dir=threadsafe_data_dir, branch=TESTDATA_BRANCH
     )  # Needed for modules where xarray is imported as `xr`
 
 
@@ -583,7 +583,7 @@ def atmosds(threadsafe_data_dir) -> xr.Dataset:
     return _open_dataset(
         threadsafe_data_dir.joinpath("atmosds.nc"),
         cache_dir=threadsafe_data_dir,
-        branch=MAIN_TESTDATA_BRANCH,
+        branch=TESTDATA_BRANCH,
     )
 
 
@@ -610,16 +610,19 @@ def gather_session_data(threadsafe_data_dir, worker_id, xdoctest_namespace):
     When running pytest with multiple workers, one worker will copy data remotely to _default_cache_dir while
     other workers wait using lockfile. Once the lock is released, all workers will copy data to their local
     threadsafe_data_dir."""
-    if worker_id == "master":
-        if not SKIP_TEST_DATA:
-            populate_testing_data(branch=MAIN_TESTDATA_BRANCH)
-    else:
-        if not SKIP_TEST_DATA:
+
+    if (
+        not _default_cache_dir.joinpath(TESTDATA_BRANCH).exists()
+        or PREFETCH_TESTING_DATA
+    ):
+        if worker_id in "master":
+            populate_testing_data(branch=TESTDATA_BRANCH)
+        else:
             _default_cache_dir.mkdir(exist_ok=True)
             test_data_being_written = FileLock(_default_cache_dir.joinpath(".lock"))
             with test_data_being_written as fl:
                 # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
-                populate_testing_data(branch=MAIN_TESTDATA_BRANCH)
+                populate_testing_data(branch=TESTDATA_BRANCH)
                 _default_cache_dir.joinpath(".data_written").touch()
             fl.acquire()
         shutil.copytree(_default_cache_dir, threadsafe_data_dir)

xclim/testing/tests/data.py

@@ -17,7 +17,7 @@
 from xclim.testing import open_dataset as _open_dataset
 from xclim.testing.utils import _default_cache_dir  # noqa
 
-MAIN_TESTDATA_BRANCH = os.getenv("MAIN_TESTDATA_BRANCH", "main")
+TESTDATA_BRANCH = os.getenv("XCLIM_TESTDATA_BRANCH", "main")
 TD = Path(__file__).parent / "data"
 
 
@@ -34,7 +34,7 @@ def generate_atmos(cache_dir: Path):
     with _open_dataset(
         "ERA5/daily_surface_cancities_1990-1993.nc",
         cache_dir=cache_dir,
-        branch=MAIN_TESTDATA_BRANCH,
+        branch=TESTDATA_BRANCH,
     ) as ds:
         tn10 = calendar.percentile_doy(ds.tasmin, per=10)
         t10 = calendar.percentile_doy(ds.tas, per=10)
@@ -60,7 +60,7 @@ def generate_atmos(cache_dir: Path):
 
 def populate_testing_data(
     temp_folder: Path | None = None,
-    branch: str = MAIN_TESTDATA_BRANCH,
+    branch: str = TESTDATA_BRANCH,
     _local_cache: Path = _default_cache_dir,
 ):
     """Perform calls to GitHub for the relevant testing data."""
@@ -112,8 +112,6 @@ def add_example_file_paths(cache_dir: Path) -> dict[str]:
     ns["path_to_tasmax_file"] = "NRCANdaily/nrcan_canada_daily_tasmax_1990.nc"
     ns["path_to_tasmin_file"] = "NRCANdaily/nrcan_canada_daily_tasmin_1990.nc"
     ns["path_to_tas_file"] = "ERA5/daily_surface_cancities_1990-1993.nc"
-    ns["path_to_multi_shape_file"] = str(TD / "multi_regions.json")
-    ns["path_to_shape_file"] = str(TD / "southern_qc_geojson.json")
     ns["path_to_ensemble_file"] = "EnsembleReduce/TestEnsReduceCriteria.nc"
 
     # For core.utils.load_module example
@@ -151,9 +149,7 @@ def add_example_file_paths(cache_dir: Path) -> dict[str]:
     atmos_file = cache_dir.joinpath("atmosds.nc")
 
     # Give access to dataset variables by name in xdoctest namespace
-    with _open_dataset(
-        atmos_file, branch=MAIN_TESTDATA_BRANCH, cache_dir=cache_dir
-    ) as ds:
+    with _open_dataset(atmos_file, branch=TESTDATA_BRANCH, cache_dir=cache_dir) as ds:
         for variable in ds.data_vars:
             ns[f"{variable}_dataset"] = ds.get(variable)