Merge pull request #267 from CSHS-CWRA/fix-conda-netcdf

Fix file-access errors with newest NetCDF4, adjust testing default settings

.github/workflows/main.yml

@@ -6,6 +6,9 @@ on:
       - master
   pull_request:
 
+env:
+  RAVENPY_TESTDATA_BRANCH: master
+
 jobs:
   black:
     name: Code linting
@@ -51,7 +54,7 @@ jobs:
         sudo apt-get install libgdal-dev
     - name: Install tox and setuptools
       run: |
-        pip install tox
+        pip install tox~=4.0
         pip install --upgrade "setuptools<65.6"
     - name: Test with tox and report coverage
       run: env GDAL_VERSION="$(gdal-config --version)" tox -e ${{ matrix.tox-env }}
@@ -90,7 +93,7 @@ jobs:
           conda list
       - name: Test RavenPy
         run: |
-          pytest --cov ravenpy
+          pytest --cov --numprocesses=logical
       - name: Report coverage
         run: coveralls --service=github
         env:

requirements_dev.txt

@@ -11,7 +11,8 @@ twine
 Click
 pytest
 pytest-cov
-pytest-xdist
+pytest-xdist>=3.2.0
+filelock
 black>=23.1.0
 isort
 pre-commit

setup.cfg

@@ -19,16 +19,17 @@ replace = "version": "{new_version}",
 test = pytest
 
 [tool:pytest]
-collect_ignore = 
+collect_ignore =
 	setup.py
-addopts = 
+addopts =
 	--color=yes
 	--verbose
-	--numprocesses=auto
-	--maxprocesses=6
+	--numprocesses=0
+	--maxprocesses=8
+    --dist=worksteal
 python_files = test_*.py
 norecursedirs = src .git bin
-filterwarnings = 
+filterwarnings =
 	ignore::UserWarning
 
 [isort]
@@ -41,7 +42,7 @@ relative_files = True
 omit = tests/*
 
 [flake8]
-exclude = 
+exclude =
 	.git,
 	docs,
 	build,
@@ -56,7 +57,7 @@ exclude =
 	.txt,
 max-line-length = 88
 max-complexity = 12
-ignore = 
+ignore =
 	C901
 	E203
 	E231
@@ -66,9 +67,9 @@ ignore =
 	F403
 	W503
 	W504
-per-file-ignores = 
+per-file-ignores =
 	tests/*:E402
-rst-roles = 
+rst-roles =
 	mod,
 	py:attr,
 	py:attribute,

tests/common.py

@@ -73,18 +73,25 @@ def _convert_2d(fn):
         "latitude": 54.4848,
         "longitude": -123.3659,
     }
-
     ds = xr.open_dataset(fn, decode_times=False).rename({"nstations": "region"})
 
+    out = xr.Dataset(
+        coords={
+            "lon": ds.lon.expand_dims("lon").squeeze("region"),
+            "lat": ds.lat.expand_dims("lat").squeeze("region"),
+            "time": ds.time,
+        }
+    )
+
     for v in ds.data_vars:
         if v not in ["lon", "lat"]:
-            ds[v] = ds[v].expand_dims("region", axis=1)
+            out[v] = ds[v].expand_dims("region", axis=1)
 
     # Add geometry feature variables
     for key, val in features.items():
-        ds[key] = xr.DataArray(name=key, data=[val], dims=("region"))
+        out[key] = xr.DataArray(name=key, data=[val], dims="region")
 
-    return ds
+    return out
 
 
 def _convert_3d(fn):

tests/conftest.py

@@ -12,13 +12,15 @@
 from ravenpy.utilities.testdata import get_file as _get_file
 from ravenpy.utilities.testdata import get_local_testdata as _get_local_testdata
 
-MAIN_TESTDATA_BRANCH = os.getenv("MAIN_TESTDATA_BRANCH", "master")
-SKIP_TEST_DATA = os.getenv("SKIP_TEST_DATA")
+from .common import _convert_2d, _convert_3d
+
+TESTDATA_BRANCH = os.getenv("RAVENPY_TESTDATA_BRANCH", "master")
+SKIP_TEST_DATA = os.getenv("RAVENPY_SKIP_TEST_DATA")
 
 
 def populate_testing_data(
     temp_folder: Optional[Path] = None,
-    branch: str = MAIN_TESTDATA_BRANCH,
+    branch: str = TESTDATA_BRANCH,
     _local_cache: Path = _default_cache_dir,
 ):
     if _local_cache.joinpath(".data_written").exists():
@@ -118,9 +120,7 @@ def threadsafe_data_dir(tmp_path_factory) -> Path:
 @pytest.fixture(scope="session")
 def get_file(threadsafe_data_dir):
     def _get_session_scoped_file(file: Union[str, Path]):
-        return _get_file(
-            file, cache_dir=threadsafe_data_dir, branch=MAIN_TESTDATA_BRANCH
-        )
+        return _get_file(file, cache_dir=threadsafe_data_dir, branch=TESTDATA_BRANCH)
 
     return _get_session_scoped_file
 
@@ -131,7 +131,7 @@ def _get_session_scoped_local_testdata(file: Union[str, Path]):
         return _get_local_testdata(
             file,
             temp_folder=threadsafe_data_dir,
-            branch=MAIN_TESTDATA_BRANCH,
+            branch=TESTDATA_BRANCH,
             _local_cache=_default_cache_dir,
         )
 
@@ -147,14 +147,14 @@ def gather_session_data(threadsafe_data_dir, worker_id):
     threadsafe_data_dir."""
     if worker_id == "master":
         if not SKIP_TEST_DATA:
-            populate_testing_data(branch=MAIN_TESTDATA_BRANCH)
+            populate_testing_data(branch=TESTDATA_BRANCH)
     else:
         if not SKIP_TEST_DATA:
             _default_cache_dir.mkdir(exist_ok=True)
             test_data_being_written = FileLock(_default_cache_dir.joinpath(".lock"))
             with test_data_being_written as fl:
                 # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
-                populate_testing_data(branch=MAIN_TESTDATA_BRANCH)
+                populate_testing_data(branch=TESTDATA_BRANCH)
                 _default_cache_dir.joinpath(".data_written").touch()
             fl.acquire()
         shutil.copytree(_default_cache_dir, threadsafe_data_dir)
@@ -181,7 +181,7 @@ def q_sim_1(threadsafe_data_dir):
     return _get_file(
         "hydro_simulations/raven-gr4j-cemaneige-sim_hmets-0_Hydrographs.nc",
         cache_dir=threadsafe_data_dir,
-        branch=MAIN_TESTDATA_BRANCH,
+        branch=TESTDATA_BRANCH,
     )
 
 
@@ -196,14 +196,53 @@ def ts_stats(q_sim_1, tmp_path):
 
 @pytest.fixture(scope="session")
 def params(ts_stats, tmp_path):
-    ds = xr.open_dataset(ts_stats)
-    name = list(ds.data_vars.keys()).pop()
-    q = ds[name]
-    p = fit(q, dist="gumbel_r")
     fn = tmp_path / "fit.nc"
-    p.to_netcdf(fn)
+
+    if not fn.exists():
+        ds = xr.open_dataset(ts_stats)
+        name = list(ds.data_vars.keys()).pop()
+        q = ds[name]
+        p = fit(q, dist="gumbel_r")
+        p.to_netcdf(fn)
+
     return fn
 
 
+@pytest.fixture(scope="session")
+def salmon(threadsafe_data_dir):
+    """A file storing a Raven streamflow simulation over one basin."""
+    return _get_file(
+        "raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc",
+        cache_dir=threadsafe_data_dir,
+        branch=TESTDATA_BRANCH,
+    )
+
+
+# Used in test_emulators.py
+@pytest.fixture(scope="session")
+def input2d(threadsafe_data_dir, salmon):
+    """Convert 1D input to 2D output by copying all the time series along a new region dimension."""
+
+    fn_out = threadsafe_data_dir / "input2d.nc"
+    if not fn_out.exists():
+        _convert_2d(salmon).to_netcdf(fn_out)
+
+    return fn_out
+
+
+# Used in test_emulators.py
+@pytest.fixture(scope="session")
+def input3d(threadsafe_data_dir, salmon):
+    """Convert 1D input to 2D output by copying all the time series along a new region dimension."""
+
+    fn_out = threadsafe_data_dir / "input3d.nc"
+    if not fn_out.exists():
+        ds = _convert_3d(salmon)
+        ds = ds.drop_vars("qobs")
+        ds.to_netcdf(fn_out)
+
+    return fn_out
+
+
 if __name__ == "__main__":
-    populate_testing_data(branch=MAIN_TESTDATA_BRANCH)
+    populate_testing_data(branch=TESTDATA_BRANCH)

tests/test_emulators.py

@@ -1,5 +1,4 @@
 import datetime as dt
-import os
 import zipfile
 from dataclasses import astuple
 from pathlib import Path
@@ -33,35 +32,10 @@
     get_average_annual_runoff,
 )
 
-from .common import _convert_2d, _convert_3d
-
 # Link to THREDDS Data Server netCDF testdata
 TDS = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/raven"
 
 
-@pytest.fixture
-def input2d(tmpdir, get_file):
-    """Convert 1D input to 2D output by copying all the time series along a new region dimension."""
-    ts = get_file(salmon_river)
-
-    ds = _convert_2d(ts)
-    fn_out = os.path.join(tmpdir, "input2d.nc")
-    ds.to_netcdf(fn_out)
-    return Path(fn_out)
-
-
-@pytest.fixture
-def input3d(tmpdir, get_file):
-    """Convert 1D input to 2D output by copying all the time series along a new region dimension."""
-    ts = get_file(salmon_river)
-
-    ds = _convert_3d(ts)
-    ds = ds.drop_vars("qobs")
-    fn_out = os.path.join(tmpdir, "input3d.nc")
-    ds.to_netcdf(fn_out)
-    return Path(fn_out)
-
-
 def test_race():
     model1 = GR4JCN()
     model1.config.rvi.suppress_output = True

tox.ini

@@ -1,7 +1,8 @@
 [tox]
+min_version = 4.0
 envlist = black, py{38,39,310}-numpy, docs
 requires =
-    pip >= 20.0
+    pip >= 21.0
     setuptools >=63.0,<65.6
 opts = --verbose
 
@@ -20,13 +21,14 @@ allowlist_externals = make
 
 [testenv]
 setenv =
-    PYTEST_ADDOPTS = "--color=yes"
+    PYTEST_ADDOPTS = --numprocesses=logical --durations=10
 passenv =
     CI
     COVERALLS_*
     GDAL_VERSION
     GITHUB_*
     LD_PRELOAD
+    RAVENPY_*
 extras = dev
 download = true
 install_command =