diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd4741cb..49e3f0a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,6 @@ jobs: run: | py.test ./tests -m "not notebooks" - - name: Run codacy-coverage-reporter uses: codacy/codacy-coverage-reporter-action@master with: diff --git a/.gitignore b/.gitignore index 692d5579..52f5aa9e 100644 --- a/.gitignore +++ b/.gitignore @@ -146,7 +146,13 @@ nlmod/bin/* !nlmod/bin/ !nlmod/bin/mp7_2_002_provisional flowchartnlmod.pptx -tests/data/ + +tests/data/* +!tests/data/**/ +tests/data/mfoutput/* +!tests/data/mfoutput/vertex/* +!tests/data/mfoutput/structured/* + docs/examples/*/ !docs/examples/data/ !docs/examples/data/chloride_hbossche.nc diff --git a/.prospector.yaml b/.prospector.yaml index b70ef805..57c8ec7c 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -26,4 +26,10 @@ pylint: - no-else-raise - too-many-arguments - too-many-locals + - too-many-branches + - too-many-statements - logging-fstring-interpolation + +mccabe: + disable: + - MC0001 \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml index 9cf35493..f229e3c0 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,9 +7,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.9" + python: "3.10" # Build documentation in the docs/ directory with Sphinx sphinx: @@ -20,7 +20,6 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: - system_packages: true install: - method: pip path: . diff --git a/README.md b/README.md index 81bd952d..32c5f17c 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,23 @@ [![PyPI version](https://badge.fury.io/py/nlmod.svg)](https://badge.fury.io/py/nlmod) [![Documentation Status](https://readthedocs.org/projects/nlmod/badge/?version=stable)](https://nlmod.readthedocs.io/en/stable/?badge=stable) -Python package with functions to process, build and visualise MODFLOW models in the Netherlands. +Python package to build, run and visualize MODFLOW 6 groundwater models in the Netherlands. -The functions in nlmod have four main objectives: +`nlmod` was built to allow users to write scripts to quickly download relevant data +from publicly available sources, and build and post-process groundwater flow and +transport models at different spatial and temporal scales to answer specific +geohydrological questions. Scripting these steps, from downloading data to building +groundwater models, makes models more reproducible and transparent. + +The functions in `nlmod` have four main objectives: 1. Create and adapt the temporal and spatial discretization of a MODFLOW model using an xarray Dataset (`nlmod.dims`). 2. Download and read data from external sources, project this data on the modelgrid and add this data to an xarray Dataset (`nlmod.read`). -3. Use data in an xarray Dataset to build modflow packages using FloPy (`nlmod.gwf` for Modflow 6 and `nlmod.modpath` for Modpath). +3. Use data in an xarray Dataset to build modflow packages for both groundwater flow and transport models using FloPy (`nlmod.sim`, `nlmod.gwf` and `nlmod.gwt` for Modflow 6 and `nlmod.modpath` for Modpath). 4. Visualise modeldata in Python (`nlmod.plot`) or GIS software (`nlmod.gis`). -More information can be found on the documentation-website: https://nlmod.readthedocs.io/. +More information can be found on the documentation-website: +https://nlmod.readthedocs.io/. ## Installation @@ -25,18 +32,43 @@ Install the module with pip: `pip install nlmod` -`nlmod` has many required dependencies: `flopy`, `xarray`, `netcdf4`, `rasterio`, `rioxarray`, `affine`, `geopandas`, `owslib`, `hydropandas`, `shapely`, `pyshp`, `rtree`, `matplotlib`, `dask` and `colorama`. On top of that there are some optional dependecies, only needed (and imported) in a single method. Examples of this are `bottleneck` (used in calculate_gxg), `geocube` (used in add_min_ahn_to_gdf), `h5netcdf` (used for hdf5 files backend in xarray). To install the nlmod with the optional dependencies use: +`nlmod` has the following required dependencies: + +* `flopy` +* `xarray` +* `netcdf4` +* `rasterio` +* `rioxarray` +* `affine` +* `geopandas` +* `owslib` +* `hydropandas` +* `shapely` +* `pyshp` +* `rtree` +* `matplotlib` +* `dask` +* `colorama` + +There are some optional dependecies, only needed (and imported) in a single method. +Examples of this are `bottleneck` (used in calculate_gxg), `geocube` (used in +add_min_ahn_to_gdf), `h5netcdf` (used for hdf5 files backend in xarray), `scikit-image` +(used in calculate_sea_coverage). To install `nlmod` with the optional dependencies use: `pip install nlmod[full]` -When using pip the dependencies are automatically installed. Some dependencies are notoriously hard to install on certain platforms. -Please see the [dependencies](https://github.com/ArtesiaWater/hydropandas#dependencies) section of the `hydropandas` package for more information on how to install these packages manually. +When using pip the dependencies are automatically installed. Some dependencies are +notoriously hard to install on certain platforms. Please see the +[dependencies](https://github.com/ArtesiaWater/hydropandas#dependencies) section of the +`hydropandas` package for more information on how to install these packages manually. ## Getting started -If you are using nlmod for the first time you need to download the MODFLOW executables. You can easily download these executables by running this Python code: +If you are using `nlmod` for the first time you need to download the MODFLOW +executables. You can easily download these executables by running this Python code: import nlmod nlmod.download_mfbinaries() -After you've downloaded the executables you can run the Jupyter Notebooks in the examples folder. These notebooks illustrate how you to use the nlmod package. +After you've downloaded the executables you can run the Jupyter Notebooks in the +examples folder. These notebooks illustrate how to use the `nlmod` package. diff --git a/docs/_static/logo_brabant_water.svg b/docs/_static/logo_brabant_water.svg new file mode 100644 index 00000000..f9c36d80 --- /dev/null +++ b/docs/_static/logo_brabant_water.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/_static/logo_evides.png b/docs/_static/logo_evides.png new file mode 100644 index 00000000..4c234baf Binary files /dev/null and b/docs/_static/logo_evides.png differ diff --git a/docs/_static/logo_hhnk.svg b/docs/_static/logo_hhnk.svg new file mode 100644 index 00000000..540127b3 --- /dev/null +++ b/docs/_static/logo_hhnk.svg @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/_static/logo_pwn.svg b/docs/_static/logo_pwn.svg new file mode 100644 index 00000000..90846f6e --- /dev/null +++ b/docs/_static/logo_pwn.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + diff --git a/docs/_static/logo_vitens.jpg b/docs/_static/logo_vitens.jpg new file mode 100644 index 00000000..a8b52263 Binary files /dev/null and b/docs/_static/logo_vitens.jpg differ diff --git a/docs/_static/logo_waternet.png b/docs/_static/logo_waternet.png new file mode 100644 index 00000000..c93da686 Binary files /dev/null and b/docs/_static/logo_waternet.png differ diff --git a/docs/examples/00_model_from_scratch.ipynb b/docs/examples/00_model_from_scratch.ipynb index ca010ed9..40579e4f 100644 --- a/docs/examples/00_model_from_scratch.ipynb +++ b/docs/examples/00_model_from_scratch.ipynb @@ -122,7 +122,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = nlmod.time.set_ds_time(ds, time=pd.Timestamp.today())" + "ds = nlmod.time.set_ds_time(ds, time=pd.Timestamp.today(), start=1)" ] }, { @@ -270,20 +270,17 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(10, 8))\n", - "\n", - "pc = nlmod.plot.data_array(\n", + "# using nlmod plotting methods\n", + "ax = nlmod.plot.map_array(\n", " head.sel(layer=0).isel(time=0),\n", - " ds=ds,\n", + " ds,\n", " cmap=\"RdYlBu\",\n", - " ax=ax,\n", - ")\n", - "nlmod.plot.modelgrid(ds, ax=ax, alpha=0.5, lw=0.5)\n", - "ax.axis(extent)\n", - "cbar = ax.figure.colorbar(pc, shrink=1.0)\n", - "cbar.set_label(\"head [m NAP]\")\n", - "ax.set_title(\"head first layer\")\n", - "fig.tight_layout()" + " colorbar_label=\"head [m NAP]\",\n", + " xlabel=\"x [km]\",\n", + " ylabel=\"y [km]\",\n", + " title=\"head first layer\",\n", + " plot_grid=True,\n", + ")" ] }, { @@ -300,6 +297,7 @@ "metadata": {}, "outputs": [], "source": [ + "# using xarray plotting methods\n", "fg = head.plot(\n", " x=\"x\",\n", " y=\"y\",\n", @@ -307,6 +305,7 @@ " col_wrap=3,\n", " cmap=\"RdYlBu\",\n", " subplot_kws={\"aspect\": \"equal\"},\n", + " cbar_kwargs={\"label\": \"head [m NAP]\"},\n", ")\n", "\n", "for iax in fg.axs.flat:\n", diff --git a/docs/examples/01_basic_model.ipynb b/docs/examples/01_basic_model.ipynb index a1ee2ef6..ca18a257 100644 --- a/docs/examples/01_basic_model.ipynb +++ b/docs/examples/01_basic_model.ipynb @@ -35,11 +35,10 @@ "source": [ "print(f\"nlmod version: {nlmod.__version__}\")\n", "\n", - "nlmod.util.get_color_logger(\"INFO\");" + "nlmod.util.get_color_logger(\"INFO\")" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -98,9 +97,7 @@ "ds = nlmod.to_model_ds(layer_model, model_name, model_ws, delr=delr, delc=delc)\n", "\n", "# add time discretisation\n", - "ds = nlmod.time.set_ds_time(\n", - " ds, start_time=start_time, steady_state=steady_state, perlen=365 * 5\n", - ")\n", + "ds = nlmod.time.set_ds_time(ds, start=start_time, steady=steady_state, perlen=365 * 5)\n", "\n", "if add_northsea:\n", " ds = nlmod.read.rws.add_northsea(ds, cachedir=cachedir)" @@ -177,7 +174,7 @@ "outputs": [], "source": [ "# add constant head cells at model boundaries\n", - "ds.update(nlmod.grid.mask_model_edge(ds, ds[\"idomain\"]))\n", + "ds.update(nlmod.grid.mask_model_edge(ds))\n", "chd = nlmod.gwf.chd(ds, gwf, mask=\"edge_mask\", head=\"starting_head\")" ] }, @@ -228,7 +225,7 @@ "metadata": {}, "outputs": [], "source": [ - "nlmod.sim.write_and_run(sim, ds, write_ds=True, nb_path=\"01_basic_model.ipynb\")" + "nlmod.sim.write_and_run(sim, ds, write_ds=True, script_path=\"01_basic_model.ipynb\")" ] }, { @@ -262,24 +259,17 @@ "metadata": {}, "outputs": [], "source": [ - "fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(14, 11))\n", + "fig, axes = nlmod.plot.get_map(ds.extent, nrows=2, ncols=2, figsize=14)\n", "ds[\"ahn\"].plot(ax=axes[0][0])\n", "ds[\"botm\"][0].plot(ax=axes[0][1])\n", - "ds[\"idomain\"][0].plot(ax=axes[1][0])\n", + "nlmod.layers.get_idomain(ds)[0].plot(ax=axes[1][0])\n", "ds[\"edge_mask\"][0].plot(ax=axes[1][1])\n", - "for axes1 in axes:\n", - " for ax in axes1:\n", - " ax.axis(\"scaled\")\n", "\n", - "fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(14, 11))\n", + "fig, axes = nlmod.plot.get_map(ds.extent, nrows=2, ncols=2, figsize=14)\n", "ds[\"bathymetry\"].plot(ax=axes[0][0])\n", "ds[\"northsea\"].plot(ax=axes[0][1])\n", "ds[\"kh\"][1].plot(ax=axes[1][0])\n", - "ds[\"recharge\"].plot(ax=axes[1][1])\n", - "\n", - "for axes1 in axes:\n", - " for ax in axes1:\n", - " ax.axis(\"scaled\")" + "ds[\"recharge\"].plot(ax=axes[1][1]);" ] } ], diff --git a/docs/examples/02_surface_water.ipynb b/docs/examples/02_surface_water.ipynb index f96a20bd..143d4eb7 100644 --- a/docs/examples/02_surface_water.ipynb +++ b/docs/examples/02_surface_water.ipynb @@ -42,7 +42,7 @@ "source": [ "print(f\"nlmod version: {nlmod.__version__}\")\n", "\n", - "nlmod.util.get_color_logger(\"INFO\");" + "nlmod.util.get_color_logger(\"INFO\")" ] }, { @@ -152,7 +152,7 @@ "outputs": [], "source": [ "f, ax = nlmod.plot.get_map(extent)\n", - "bgt.plot(\"bronhouder\", legend=True, ax=ax);" + "bgt.plot(\"bronhouder\", legend=True, ax=ax)" ] }, { @@ -293,14 +293,8 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(10, 8))\n", - "ax.set_aspect(\"equal\", adjustable=\"box\")\n", - "sfw.plot(ax=ax, column=\"stage\", legend=True)\n", - "ax.grid(True)\n", - "ax.set_xlabel(\"X (m RD)\")\n", - "ax.set_ylabel(\"Y (m RD)\")\n", - "plt.yticks(rotation=90, va=\"center\")\n", - "fig.tight_layout()" + "fig, ax = nlmod.plot.get_map(extent)\n", + "sfw.plot(ax=ax, column=\"stage\", legend=True)" ] }, { @@ -357,7 +351,7 @@ "ds = nlmod.to_model_ds(layer_model, model_name, model_ws, delr=delr, delc=delc)\n", "\n", "# create model time dataset\n", - "ds = nlmod.time.set_ds_time(ds, start_time=start_time, steady_state=True)\n", + "ds = nlmod.time.set_ds_time(ds, start=start_time, steady=True, perlen=1)\n", "\n", "ds" ] @@ -446,14 +440,9 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(10, 8))\n", - "ax.set_aspect(\"equal\", adjustable=\"box\")\n", + "fig, ax = nlmod.plot.get_map(extent)\n", "sfw_grid.plot(ax=ax, column=\"cellid\")\n", - "gwf.modelgrid.plot(ax=ax, linewidth=0.5, color=\"k\")\n", - "xmin, xmax, ymin, ymax = extent\n", - "offset = 100\n", - "ax.set_xlim(xmin - offset, xmax + offset)\n", - "ax.set_ylim(ymin - offset, ymax + offset);" + "nlmod.plot.modelgrid(ds, ax=ax, lw=0.2)" ] }, { @@ -494,14 +483,17 @@ "source": [ "fig, ax = plt.subplots(1, 1, figsize=(10, 8))\n", "sfw_grid.loc[mask].plot(\n", - " column=\"identificatie\", legend=True, ax=ax, legend_kwds={\"loc\": \"upper left\"}\n", + " column=\"identificatie\",\n", + " legend=True,\n", + " ax=ax,\n", + " legend_kwds={\"loc\": \"lower left\", \"ncol\": 2, \"fontsize\": \"x-small\"},\n", ")\n", "xlim = ax.get_xlim()\n", "ylim = ax.get_ylim()\n", "gwf.modelgrid.plot(ax=ax)\n", "ax.set_xlim(xlim[0], xlim[0] + ds.delr * 1.1)\n", "ax.set_ylim(ylim)\n", - "ax.set_title(f\"Surface water shapes in cell: {cid}\");" + "ax.set_title(f\"Surface water shapes in cell: {cid}\")" ] }, { @@ -691,6 +683,7 @@ "metadata": {}, "outputs": [], "source": [ + "# use flopy plotting methods\n", "fig, ax = plt.subplots(1, 1, figsize=(10, 8), constrained_layout=True)\n", "mv = flopy.plot.PlotMapView(model=gwf, ax=ax, layer=0)\n", "mv.plot_bc(\"RIV\")" @@ -713,7 +706,7 @@ "metadata": {}, "outputs": [], "source": [ - "nlmod.sim.write_and_run(sim, ds, write_ds=True, nb_path=\"02_surface_water.ipynb\")" + "nlmod.sim.write_and_run(sim, ds, write_ds=True, script_path=\"02_surface_water.ipynb\")" ] }, { @@ -751,13 +744,17 @@ "metadata": {}, "outputs": [], "source": [ - "ilay = 0\n", - "fig, ax = plt.subplots(1, 1, figsize=(10, 8), constrained_layout=True)\n", - "mv = flopy.plot.PlotMapView(model=gwf, ax=ax, layer=ilay)\n", - "qm = mv.plot_array(head[-1], cmap=\"RdBu\") # last timestep\n", - "mv.plot_ibound() # plot inactive cells in red\n", - "fig.colorbar(qm, shrink=1.0)\n", - "ax.set_title(f\"Heads top-view, layer {ilay}\");" + "# using nlmod plotting methods\n", + "ax = nlmod.plot.map_array(\n", + " head,\n", + " ds,\n", + " ilay=0,\n", + " iper=0,\n", + " plot_grid=True,\n", + " title=\"Heads top-view\",\n", + " cmap=\"RdBu\",\n", + " colorbar_label=\"head [m NAP]\",\n", + ")" ] }, { @@ -775,16 +772,29 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(10, 3), constrained_layout=True)\n", - "xs = flopy.plot.PlotCrossSection(\n", - " model=gwf, ax=ax, line={\"column\": gwf.modelgrid.ncol // 2}\n", - ")\n", + "# using flopy plotting methods\n", + "col = gwf.modelgrid.ncol // 2\n", + "\n", + "fig, ax = plt.subplots(1, 1, figsize=(10, 3))\n", + "xs = flopy.plot.PlotCrossSection(model=gwf, ax=ax, line={\"column\": col})\n", "qm = xs.plot_array(head[-1], cmap=\"RdBu\") # last timestep\n", "xs.plot_ibound() # plot inactive cells in red\n", - "fig.colorbar(qm, shrink=1.0)\n", - "col = gwf.modelgrid.ncol // 2\n", - "ax.set_title(f\"Cross-section along column {col}\");" + "xs.plot_grid(lw=0.25, color=\"k\")\n", + "ax.set_ylim(bottom=-150)\n", + "ax.set_ylabel(\"elevation [m NAP]\")\n", + "ax.set_xlabel(\"distance along cross-section [m]\")\n", + "ax.set_title(f\"Cross-section along column {col}\")\n", + "cbar = fig.colorbar(qm, shrink=1.0)\n", + "cbar.set_label(\"head [m NAP]\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e5833d6", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/examples/03_local_grid_refinement.ipynb b/docs/examples/03_local_grid_refinement.ipynb index 79d7f9cd..60e5606f 100644 --- a/docs/examples/03_local_grid_refinement.ipynb +++ b/docs/examples/03_local_grid_refinement.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -40,11 +39,10 @@ "source": [ "print(f\"nlmod version: {nlmod.__version__}\")\n", "\n", - "nlmod.util.get_color_logger(\"INFO\");" + "nlmod.util.get_color_logger(\"INFO\")" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -98,11 +96,10 @@ "# add time discretisation\n", "ds = nlmod.time.set_ds_time(\n", " ds,\n", - " start_time=start_time,\n", - " steady_state=steady_state,\n", + " start=start_time,\n", + " steady=steady_state,\n", " steady_start=steady_start,\n", - " transient_timesteps=transient_timesteps,\n", - " perlen=perlen,\n", + " perlen=[perlen] * (transient_timesteps + 1),\n", ")" ] }, @@ -116,7 +113,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -191,7 +187,7 @@ "drn = nlmod.gwf.surface_drain_from_ds(ds, gwf, resistance=10.0)\n", "\n", "# add constant head cells at model boundaries\n", - "ds.update(nlmod.grid.mask_model_edge(ds, ds[\"idomain\"]))\n", + "ds.update(nlmod.grid.mask_model_edge(ds))\n", "chd = nlmod.gwf.chd(ds, gwf, mask=\"edge_mask\", head=\"starting_head\")" ] }, @@ -219,7 +215,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -233,12 +228,11 @@ "outputs": [], "source": [ "nlmod.sim.write_and_run(\n", - " sim, ds, write_ds=True, nb_path=\"03_local_grid_refinement.ipynb\"\n", + " sim, ds, write_ds=True, script_path=\"03_local_grid_refinement.ipynb\"\n", ")" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -272,7 +266,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -289,7 +282,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -308,18 +300,17 @@ "\n", "nlmod.plot.data_array(ds[\"ahn\"], ds, ax=axes[0][0])\n", "nlmod.plot.data_array(ds[\"botm\"][0], ds, ax=axes[0][1])\n", - "nlmod.plot.data_array(ds[\"idomain\"][0], ds, ax=axes[1][0])\n", + "nlmod.plot.data_array(nlmod.layers.get_idomain(ds)[0], ds, ax=axes[1][0])\n", "nlmod.plot.data_array(ds[\"edge_mask\"][0], ds, ax=axes[1][1])\n", "\n", "fig, axes = nlmod.plot.get_map(extent, nrows=2, ncols=2, figsize=(14, 11))\n", "nlmod.plot.data_array(ds[\"bathymetry\"], ds, ax=axes[0][0])\n", "nlmod.plot.data_array(ds[\"northsea\"], ds, ax=axes[0][1])\n", "nlmod.plot.data_array(ds[\"kh\"][1], ds, ax=axes[1][0])\n", - "nlmod.plot.data_array(ds[\"recharge\"][0], ds, ax=axes[1][1])" + "nlmod.plot.data_array(ds[\"recharge\"][0], ds, ax=axes[1][1]);" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/docs/examples/06_gridding_vector_data.ipynb b/docs/examples/06_gridding_vector_data.ipynb index 11455287..1f999097 100644 --- a/docs/examples/06_gridding_vector_data.ipynb +++ b/docs/examples/06_gridding_vector_data.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -33,7 +32,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -57,7 +55,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -116,7 +113,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -124,7 +120,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -165,7 +160,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -183,12 +177,11 @@ "sim = nlmod.sim.sim(ds)\n", "gwf = nlmod.gwf.gwf(ds, sim)\n", "dis = nlmod.gwf.dis(ds, gwf)\n", - "da1 = nlmod.grid.gdf_to_da(\n", - " point_gdf, ds, column=\"values\", agg_method=\"nearest\"\n", - ")\n", + "da1 = nlmod.grid.gdf_to_da(point_gdf, ds, column=\"values\", agg_method=\"nearest\")\n", "da2 = xr.DataArray(np.nan, dims=(\"y\", \"x\"), coords={\"y\": ds.y, \"x\": ds.x})\n", - "da2.values = nlmod.grid.interpolate_gdf_to_array(point_gdf, gwf, field='values', \n", - "method='linear')\n", + "da2.values = nlmod.grid.interpolate_gdf_to_array(\n", + " point_gdf, gwf, field=\"values\", method=\"linear\"\n", + ")\n", "\n", "vmin = min(da1.min(), da2.min())\n", "vmax = max(da1.max(), da2.max())\n", @@ -209,7 +202,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -250,7 +242,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -291,7 +282,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -329,7 +319,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -375,7 +364,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -412,14 +400,11 @@ "# add a nodata value\n", "ds.attrs[\"nodata\"] = -999\n", "\n", - "# create an idomain of ones except for the first cell which is zero\n", - "idomain = np.ones((ds.dims[\"layer\"], ds.dims[\"y\"], ds.dims[\"x\"]))\n", - "idomain[0, 0, 0] = 0\n", - "ds[\"idomain\"] = (\"layer\", \"y\", \"x\"), idomain" + "# set the thickness of the first cell to 0, so this cell will become inactive\n", + "ds[\"top\"][0, 0] = ds[\"botm\"][0, 0, 0]" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -442,7 +427,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -466,7 +450,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -502,7 +485,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -524,7 +506,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -541,14 +522,14 @@ "outputs": [], "source": [ "# create a reclist with col1 (str), col2 (DataArray), col3 (int)\n", + "idomain = nlmod.layers.get_idomain(ds)\n", "reclist5 = nlmod.grid.da_to_reclist(\n", - " ds, mask3d, col1=ds[\"idomain\"], col2=\"da1\", col3=9, layer=0, only_active_cells=False\n", + " ds, mask3d, col1=idomain, col2=\"da1\", col3=9, layer=0, only_active_cells=False\n", ")\n", "reclist5" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -575,22 +556,8 @@ } ], "metadata": { - "kernelspec": { - "display_name": "nlmod", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.4" + "name": "python" } }, "nbformat": 4, diff --git a/docs/examples/07_resampling.ipynb b/docs/examples/07_resampling.ipynb index 7edaeaca..093e046b 100644 --- a/docs/examples/07_resampling.ipynb +++ b/docs/examples/07_resampling.ipynb @@ -224,7 +224,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Rectangular Bivariate Spline (not yet included in nlmod)" + "### Rectangular Bivariate Spline\n", + "\n", + "*Note: not yet included as a method in nlmod*" ] }, { @@ -251,7 +253,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Rectangular Bivariate Spline with nans (not yet included in nlmod)" + "### Rectangular Bivariate Spline with nans\n", + "\n", + "*Note: not yet included as a method in nlmod*" ] }, { diff --git a/docs/examples/08_gis.ipynb b/docs/examples/08_gis.ipynb index 37d56bde..cde523a3 100644 --- a/docs/examples/08_gis.ipynb +++ b/docs/examples/08_gis.ipynb @@ -71,7 +71,7 @@ "outputs": [], "source": [ "ds_struc = xr.load_dataset(\n", - " os.path.join(model_ws, \"cache\", f\"{model_name}.nc\"), mask_and_scale=False\n", + " os.path.join(model_ws, f\"{model_name}.nc\"), mask_and_scale=False\n", ")\n", "ds_struc" ] @@ -111,7 +111,7 @@ "outputs": [], "source": [ "ds_vert = xr.load_dataset(\n", - " os.path.join(model_ws, \"cache\", f\"{model_name}.nc\"), mask_and_scale=False\n", + " os.path.join(model_ws, f\"{model_name}.nc\"), mask_and_scale=False\n", ")\n", "\n", "# get modelgrid\n", @@ -217,7 +217,7 @@ "source": [ "## Export griddata\n", "\n", - "The model data can be exported to a netcdf file that can be visualised in Qgis. For a structured model the standard model dataset (xarray.Dataset) can be exported to a netdf file. For a vertex model you have to convert the model dataset to a certain format before you can write it to a netcdf and read it with Qgis. With the code below we export the vertex model dataset to a netcdf file ('model_qgis.nc') that can be read using Qgis. For more background, see this discussion https://github.com/ArtesiaWater/nlmod/issues/14. " + "The model data can be exported to a netcdf file that can be visualised in Qgis. For a structured model the standard model dataset (xarray.Dataset) can be exported to a netdf file. For a vertex model you have to convert the model dataset to a certain format before you can write it to a netcdf and read it with Qgis. With the code below we export the vertex model dataset to a netcdf file ('model_qgis.nc') that can be read using Qgis." ] }, { diff --git a/docs/examples/09_schoonhoven.ipynb b/docs/examples/09_schoonhoven.ipynb index e07f069c..d915b077 100644 --- a/docs/examples/09_schoonhoven.ipynb +++ b/docs/examples/09_schoonhoven.ipynb @@ -41,7 +41,7 @@ "source": [ "print(f\"nlmod version: {nlmod.__version__}\")\n", "\n", - "nlmod.util.get_color_logger(\"INFO\");" + "nlmod.util.get_color_logger(\"INFO\")" ] }, { @@ -226,7 +226,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = nlmod.time.set_ds_time(ds, time=time)" + "ds = nlmod.time.set_ds_time(ds, time=time, start=3652)" ] }, { @@ -338,6 +338,7 @@ " \"identificatie\"\n", "].isin(ids_oude_haven)\n", "lakes = bgt_grid[mask].copy()\n", + "lakes[\"name\"] = \"\"\n", "lakes.loc[lakes[\"identificatie\"].isin(ids_grote_gracht), \"name\"] = \"grote gracht\"\n", "lakes.loc[lakes[\"identificatie\"].isin(ids_oude_haven), \"name\"] = \"oude haven\"\n", "bgt_grid = bgt_grid[~mask]" @@ -381,7 +382,7 @@ "metadata": {}, "outputs": [], "source": [ - "drn = nlmod.gwf.surface_water.gdf_to_seasonal_pkg(bgt_grid, gwf, ds);" + "drn = nlmod.gwf.surface_water.gdf_to_seasonal_pkg(bgt_grid, gwf, ds)" ] }, { @@ -425,7 +426,7 @@ "] = 1.0 # overstort hoogte\n", "\n", "# add lake to groundwaterflow model\n", - "nlmod.gwf.lake_from_gdf(gwf, lakes, ds, boundname_column=\"name\");" + "nlmod.gwf.lake_from_gdf(gwf, lakes, ds, boundname_column=\"name\")" ] }, { @@ -491,13 +492,18 @@ "metadata": {}, "outputs": [], "source": [ - "f, ax = nlmod.plot.get_map(extent)\n", "norm = matplotlib.colors.Normalize(-2.5, 0.0)\n", - "pc = nlmod.plot.data_array(\n", - " head.sel(layer=\"HLc\").mean(\"time\"), ds=ds, edgecolor=\"k\", norm=norm\n", + "\n", + "\n", + "pc = nlmod.plot.map_array(\n", + " head.sel(layer=\"HLc\").mean(\"time\"),\n", + " ds,\n", + " norm=norm,\n", + " colorbar=True,\n", + " colorbar_label=\"head [m NAP]\",\n", + " title=\"mean head\",\n", ")\n", - "cbar = nlmod.plot.colorbar_inside(pc)\n", - "bgt.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\")" + "bgt.plot(ax=pc.axes, edgecolor=\"k\", facecolor=\"none\");" ] }, { @@ -517,14 +523,15 @@ "source": [ "x = 118228.0\n", "line = [(x, 439000), (x, 442000)]\n", + "\n", "f, ax = plt.subplots(figsize=(10, 6))\n", - "ax.grid()\n", "dcs = DatasetCrossSection(ds, line, ax=ax, zmin=-100.0, zmax=10.0)\n", "pc = dcs.plot_array(head.mean(\"time\"), norm=norm, head=head.mean(\"time\"))\n", + "\n", "# add labels with layer names\n", - "cbar = nlmod.plot.colorbar_inside(pc, bounds=[0.05, 0.05, 0.02, 0.9])\n", + "cbar = nlmod.plot.colorbar_inside(pc)\n", "dcs.plot_grid()\n", - "dcs.plot_layers(alpha=0.0, min_label_area=1000)\n", + "dcs.plot_layers(colors=\"none\", min_label_area=1000)\n", "f.tight_layout(pad=0.0)" ] }, @@ -546,7 +553,9 @@ "x = 118228\n", "y = 439870\n", "head_point = nlmod.gwf.get_head_at_point(head, x=x, y=y, ds=ds)\n", - "head_point.plot.line(hue=\"layer\", size=10);" + "fig, ax = plt.subplots(1, 1, figsize=(10, 8))\n", + "handles = head_point.plot.line(ax=ax, hue=\"layer\")\n", + "ax.set_ylabel(\"head [m NAP]\")" ] }, { @@ -566,17 +575,17 @@ "source": [ "df = pd.read_csv(os.path.join(model_ws, \"lak_STAGE.csv\"), index_col=0)\n", "df.index = ds.time.values\n", - "ax = df.plot(figsize=(10, 3));" + "ax = df.plot(figsize=(10, 3))" ] }, { - "cell_type": "raw", + "cell_type": "markdown", "id": "9355de12", "metadata": { "raw_mimetype": "text/x-python" }, "source": [ - "### Compare with BRO measurements" + "## Compare with BRO measurements" ] }, { @@ -586,35 +595,8 @@ "raw_mimetype": "text/x-python" }, "source": [ - "fname_pklz = os.path.join(ds.cachedir, 'oc_bro.pklz')\n", - "if os.path.exists(fname_pklz):\n", - " oc = pd.read_pickle(fname_pklz)\n", - "else:\n", - " oc = hpd.read_bro(extent=ds.extent, name='BRO', tmin=ds.time.values.min(), tmax=ds.time.values.max(), )\n", - " oc.to_pickle(fname_pklz)" - ] - }, - { - "cell_type": "raw", - "id": "06fdd4cc-a15e-485e-b12e-fda9a464d30c", - "metadata": { - "raw_mimetype": "text/x-python" - }, - "source": [ - "# get modellayers\n", - "oc['modellayer'] = oc.gwobs.get_modellayers(ds=ds)" - ] - }, - { - "cell_type": "raw", - "id": "7831eddc-3b69-4cc5-b1c0-7ee09732a2f9", - "metadata": { - "raw_mimetype": "text/x-python" - }, - "source": [ - "# get modelled head at measurement points\n", - "ds['heads'] = nlmod.gwf.get_heads_da(ds)\n", - "oc_modflow = hpd.read_modflow(oc, gwf, ds['heads'].values, ds.time.values)" + "oc = nlmod.read.bro.get_bro(extent, cachedir=cachedir, cachename=\"bro\")\n", + "oc_mod = nlmod.read.bro.add_modelled_head(oc, gwf, ds=ds)" ] }, { @@ -624,22 +606,8 @@ "raw_mimetype": "text/x-python" }, "source": [ - "# add modelled head to measured heads\n", - "obs_list_map = []\n", - "for gld in oc.index:\n", - " o = oc.loc[gld,'obs'].resample('D').last().sort_index()\n", - " modelled = oc_modflow.loc[gld, 'obs']\n", - " modelled = hpd.GroundwaterObs(modelled.rename(columns={0: 'values'}), name=f'{o.name}_mod_lay{oc.loc[gld,\"modellayer\"]}', x=o.x, y=o.y, \n", - " tube_nr=o.tube_nr+1,screen_top=o.screen_top, screen_bottom=o.screen_bottom, \n", - " tube_top=o.tube_top, monitoring_well=o.monitoring_well, source='MODFLOW', unit= 'm NAP',\n", - " ground_level=o.ground_level, metadata_available=o.metadata_available)\n", - " obs_list_map.append(o)\n", - " obs_list_map.append(modelled)\n", - "\n", - "oc_map = hpd.ObsCollection.from_list(obs_list_map, name='meting+model')\n", - "\n", "# create interactive map\n", - "oc_map.plots.interactive_map(os.path.join(ds.figdir, 'iplots'))" + "oc_mod.plots.interactive_map(\"figures\", add_screen_to_legend=True)" ] }, { @@ -659,6 +627,7 @@ "outputs": [], "source": [ "layer = \"HLc\"\n", + "\n", "f, axes = nlmod.plot.get_map(extent, nrows=2, ncols=2)\n", "variables = [\"top\", \"kh\", \"botm\", \"kv\"]\n", "for i, variable in enumerate(variables):\n", @@ -723,7 +692,7 @@ "outputs": [], "source": [ "# run modpath model\n", - "nlmod.modpath.write_and_run(mpf, nb_path=\"10_modpath.ipynb\")" + "nlmod.modpath.write_and_run(mpf, script_path=\"10_modpath.ipynb\")" ] }, { @@ -776,7 +745,7 @@ "line = ax.add_collection(lc)\n", "nlmod.plot.colorbar_inside(line, label=\"Travel time (years)\")\n", "\n", - "bgt.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\");" + "bgt.plot(ax=ax, edgecolor=\"k\", facecolor=\"none\")" ] }, { diff --git a/docs/examples/10_modpath.ipynb b/docs/examples/10_modpath.ipynb index d29b43ca..0d0b2b33 100644 --- a/docs/examples/10_modpath.ipynb +++ b/docs/examples/10_modpath.ipynb @@ -63,7 +63,7 @@ "model_ws = \"ijmuiden\"\n", "model_name = \"IJm_planeten\"\n", "\n", - "ds = xr.open_dataset(os.path.join(model_ws, \"cache\", f\"{model_name}.nc\"))" + "ds = xr.open_dataset(os.path.join(model_ws, f\"{model_name}.nc\"))" ] }, { @@ -128,7 +128,7 @@ "outputs": [], "source": [ "# run modpath model\n", - "nlmod.modpath.write_and_run(mpf, nb_path=\"10_modpath.ipynb\")" + "nlmod.modpath.write_and_run(mpf, script_path=\"10_modpath.ipynb\")" ] }, { @@ -228,7 +228,7 @@ "outputs": [], "source": [ "# run modpath model\n", - "nlmod.modpath.write_and_run(mpf, nb_path=\"10_modpath.ipynb\")" + "nlmod.modpath.write_and_run(mpf, script_path=\"10_modpath.ipynb\")" ] }, { @@ -332,7 +332,7 @@ "outputs": [], "source": [ "# run modpath model\n", - "nlmod.modpath.write_and_run(mpf, nb_path=\"10_modpath.ipynb\")" + "nlmod.modpath.write_and_run(mpf, script_path=\"10_modpath.ipynb\")" ] }, { @@ -440,7 +440,7 @@ "outputs": [], "source": [ "# run modpath model\n", - "nlmod.modpath.write_and_run(mpf, nb_path=\"10_modpath.ipynb\")" + "nlmod.modpath.write_and_run(mpf, script_path=\"10_modpath.ipynb\")" ] }, { diff --git a/docs/examples/11_grid_rotation.ipynb b/docs/examples/11_grid_rotation.ipynb index de54f199..e522cc55 100644 --- a/docs/examples/11_grid_rotation.ipynb +++ b/docs/examples/11_grid_rotation.ipynb @@ -83,7 +83,7 @@ "# this period will determine the recharge-value and the ratio between summer and\n", "# winter stage of the surface water.\n", "\n", - "ds = nlmod.time.set_ds_time(ds, time=\"2023-1-1\")" + "ds = nlmod.time.set_ds_time(ds, time=\"2023-1-1\", start=\"2013\")" ] }, { @@ -146,7 +146,7 @@ "pc = nlmod.plot.data_array(\n", " ds[\"ahn\"], ds=ds, ax=axes[1], rotated=True, norm=norm, edgecolor=\"face\"\n", ")\n", - "nlmod.plot.colorbar_inside(pc, ax=axes[1])" + "nlmod.plot.colorbar_inside(pc, ax=axes[1]);" ] }, { diff --git a/docs/examples/12_layer_generation.ipynb b/docs/examples/12_layer_generation.ipynb index f01a2ec4..d11e5ae3 100644 --- a/docs/examples/12_layer_generation.ipynb +++ b/docs/examples/12_layer_generation.ipynb @@ -8,15 +8,15 @@ "# Generating model datasets\n", "\n", "This notebook contains two workflows to generate a model dataset and add layer\n", - "infomration to it:\n", + "information to it:\n", "\n", - "- Get data\n", - "- Regrid\n", + "- Get data (download or read from file)\n", + "- Regrid (convert data to match your desired grid)\n", "\n", "or\n", "\n", - "- Create grid\n", - "- Add data to dataset" + "- Create grid (define your desired grid)\n", + "- Add data to dataset (add data based on this grid)" ] }, { @@ -162,7 +162,7 @@ "id": "98b6d820", "metadata": {}, "source": [ - "When we plot the model top, we now see that in the cells with an equal resolution to that of Regis (or smaller) no infomration is lost is lost." + "When we plot the model top, we now see that in the cells with an equal resolution to that of Regis (or smaller) no information is lost." ] }, { diff --git a/docs/examples/13_plot_methods.ipynb b/docs/examples/13_plot_methods.ipynb index 0b69f2d0..ebc79dcc 100644 --- a/docs/examples/13_plot_methods.ipynb +++ b/docs/examples/13_plot_methods.ipynb @@ -7,13 +7,25 @@ "source": [ "# Plot methods in nlmod\n", "\n", - "This notebook shows the plot methods that are available in nlmod. Most plot\n", - "methods use a model Dataset as input, which is an xarray Dataset with some\n", - "required variables and attributes.\n", + "This notebook shows different methods of plotting data with nlmod. \n", "\n", - "There are some plot methods in flopy as well, which require a flopy modelgrid.\n", - "This notebook shows the plot methods in nlmod and flopy create similar plots,\n", - "so the user can choose the methods that best fit their needs." + "There are many ways to plot data and it depends on the type of data and plot which of\n", + "these method is the most convenient:\n", + "- using `nlmod.plot` utilities\n", + "- using `flopy` plot methods\n", + "- using `xarray` plot methods\n", + "\n", + "The default plot methods in nlmod use a model Dataset as input (this is an xarray\n", + "Dataset with some required variables and attributes). These plotting methods are\n", + "accessible through `nlmod.plot`.\n", + "\n", + "Flopy contains its own plotting utilities and nlmod contains some wrapper functions that\n", + "use flopy's plotting utilities under the hood. These require a flopy modelgrid or model\n", + "object. These plotting methods are accessible through `nlmod.plot.flopy`.\n", + "\n", + "Finally, xarray also allows plotting of data with `.plot()`. This is used in a few\n", + "cases in this notebook but for more detailed information, refer to the \n", + "[xarray documentation](https://xarray.pydata.org/en/v2023.08.0/gallery.html)." ] }, { @@ -61,8 +73,8 @@ "source": [ "model_name = \"Schoonhoven\"\n", "model_ws = \"schoonhoven\"\n", - "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)\n", - "ds = xr.open_dataset(os.path.join(cachedir, f\"{model_name}.nc\"))\n", + "ds = xr.open_dataset(os.path.join(model_ws, f\"{model_name}.nc\"))\n", + "\n", "# add calculated heads\n", "ds[\"head\"] = nlmod.gwf.get_heads_da(ds)\n", "ds" @@ -151,7 +163,7 @@ "# plot using flopy\n", "pcs = flopy.plot.PlotCrossSection(modelgrid=modelgrid, line={\"line\": line}, ax=ax[1])\n", "pcs.plot_array(ds[\"kh\"])\n", - "pcs.ax.set_ylim((zmin, zmax))" + "pcs.ax.set_ylim((zmin, zmax));" ] }, { @@ -173,7 +185,7 @@ "dcs = DatasetCrossSection(ds, line=line, zmin=-200, zmax=10, ax=ax)\n", "colors = nlmod.read.regis.get_legend()\n", "dcs.plot_layers(colors=colors, min_label_area=1000)\n", - "dcs.plot_grid(vertical=False, linewidth=0.5)" + "dcs.plot_grid(vertical=False, linewidth=0.5);" ] }, { diff --git a/docs/examples/14_stromingen_example.ipynb b/docs/examples/14_stromingen_example.ipynb index ee4e32e6..0dea2056 100644 --- a/docs/examples/14_stromingen_example.ipynb +++ b/docs/examples/14_stromingen_example.ipynb @@ -164,7 +164,7 @@ "source": [ "# set model time settings\n", "t = date_range(tmin, tmax, freq=freq)\n", - "ds = nlmod.time.set_ds_time(ds, time=t, steady_start=True)\n", + "ds = nlmod.time.set_ds_time(ds, start=3652, time=t, steady_start=True)\n", "\n", "# build the modflow6 gwf model\n", "gwf = nlmod.gwf.ds_to_gwf(ds)" @@ -291,8 +291,12 @@ "outputs": [], "source": [ "# plot on map\n", - "f, ax = nlmod.plot.get_map(extent, background=True)\n", - "nlmod.plot.data_array(head.sel(layer=\"PZWAz3\").mean(dim=\"time\"), ds, alpha=0.25, ax=ax);" + "ax = nlmod.plot.map_array(\n", + " head.sel(layer=\"PZWAz3\").mean(dim=\"time\"),\n", + " ds,\n", + " alpha=0.25,\n", + " background=True,\n", + ")" ] }, { @@ -312,9 +316,7 @@ "gxg = nlmod.gwf.calculate_gxg(head.sel(layer=\"HLc\"))\n", "\n", "# plot on map\n", - "f, ax = nlmod.plot.get_map(extent)\n", - "pc = nlmod.plot.data_array(gxg[\"ghg\"], ds)\n", - "nlmod.plot.colorbar_inside(pc);" + "pc = nlmod.plot.map_array(gxg[\"ghg\"], ds)" ] }, { diff --git a/docs/examples/15_geotop.ipynb b/docs/examples/15_geotop.ipynb index 56af7e21..60785808 100644 --- a/docs/examples/15_geotop.ipynb +++ b/docs/examples/15_geotop.ipynb @@ -106,7 +106,7 @@ "metadata": {}, "outputs": [], "source": [ - "gt = nlmod.read.geotop.get_geotop_raw_within_extent(extent, drop_probabilities=False)\n", + "gt = nlmod.read.geotop.get_geotop(extent, probabilities=True)\n", "gt" ] }, @@ -258,7 +258,9 @@ "metadata": {}, "source": [ "## Aggregate voxels to GeoTOP-layers\n", - "In the previous example we have added geodrological properties to the voxels of GeoTOP and plotted them. The layers of a groundwatermodel generally are thicker than the thickness of the voxels (0.5 m). Therefore we need to aggregate the voxel-data into the layers of the model. We show this process by using the stratigraphy-data of GeoTOP to form a layer model." + "In the previous example we have added geodrological properties to the voxels of GeoTOP and plotted them. The layers of a groundwatermodel generally are thicker than the thickness of the voxels (0.5 m). Therefore we need to aggregate the voxel-data into the layers of the model. We show this process by using the stratigraphy-data of GeoTOP to form a layer model, using the method `nlmod.read.geotop.to_model_layers`.\n", + "\n", + "When values for `kh` and `kv` are present in `gt`, this method also calculates the geohydrological properties of the layer model with the method `nlmod.read.geotop.aggregate_to_ds`. The method calculates the combined horizontal transmissivity, and the combined vertical resistance of all (parts of) voxels in a layer, and calculates a `kh` and `kv` value from this transmissivity and resistance." ] }, { @@ -268,28 +270,10 @@ "metadata": {}, "outputs": [], "source": [ - "gtl = nlmod.read.geotop.convert_geotop_to_ml_layers(gt)\n", + "gtl = nlmod.read.geotop.to_model_layers(gt)\n", "gtl" ] }, - { - "cell_type": "markdown", - "id": "a558bad3", - "metadata": {}, - "source": [ - "We then add geohydrological properties to this layer model using the method `nlmod.read.geotop.aggregate_to_ds`. The method calculates the combined horizontal transmissivity, and the combined vertical resistance of all (parts of) voxels in a layer, and calculates a kh and kv value from this transmissivity and resistance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ffbe6f8", - "metadata": {}, - "outputs": [], - "source": [ - "gtl = nlmod.read.geotop.aggregate_to_ds(gt, gtl)" - ] - }, { "cell_type": "markdown", "id": "6f328978", diff --git a/docs/examples/16_groundwater_transport.ipynb b/docs/examples/16_groundwater_transport.ipynb index 411b37d7..6b1c8d93 100644 --- a/docs/examples/16_groundwater_transport.ipynb +++ b/docs/examples/16_groundwater_transport.ipynb @@ -22,6 +22,7 @@ "source": [ "# import packages\n", "import nlmod\n", + "import pandas as pd\n", "import xarray as xr\n", "import flopy as fp\n", "import matplotlib.pyplot as plt\n", @@ -64,7 +65,9 @@ "transport = True\n", "\n", "start_time = \"2010-1-1\"\n", - "starting_head = 1.0" + "starting_head = 1.0\n", + "\n", + "municipalities = nlmod.read.boundaries.get_municipalities(extent=extent_hbossche)" ] }, { @@ -128,23 +131,70 @@ "# add time discretisation\n", "ds = nlmod.time.set_ds_time(\n", " ds,\n", - " start_time=start_time,\n", - " steady_state=False,\n", + " start=start_time,\n", + " steady=False,\n", " steady_start=True,\n", - " steady_start_perlen=1,\n", - " transient_timesteps=10,\n", - " perlen=365.0,\n", + " perlen=[365.0] * 10,\n", ")\n", "\n", - "if add_northsea:\n", - " ds = nlmod.read.rws.add_northsea(ds, cachedir=cachedir)\n", - "\n", "if ds.transport == 1:\n", " ds = nlmod.gwt.prepare.set_default_transport_parameters(\n", " ds, transport_type=\"chloride\"\n", " )" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# We download the digital terrain model (AHN4)\n", + "ahn = nlmod.read.ahn.get_ahn4(ds.extent)\n", + "# calculate the average surface level in each cell\n", + "ds[\"ahn\"] = nlmod.resample.structured_da_to_ds(ahn, ds, method=\"average\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# We download the surface level below the sea by downloading the vaklodingen\n", + "vaklodingen = nlmod.read.jarkus.get_dataset_jarkus(\n", + " extent_hbossche,\n", + " kind=\"vaklodingen\",\n", + " cachedir=cachedir,\n", + " cachename=\"vaklodingen.nc\",\n", + ")\n", + "# calculate the average surface level in each cell\n", + "ds[\"vaklodingen\"] = nlmod.resample.structured_da_to_ds(\n", + " vaklodingen[\"z\"], ds, method=\"average\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# calculate a new top from ahn and vaklodingen\n", + "new_top = ds[\"ahn\"].where(~ds[\"ahn\"].isnull(), ds[\"vaklodingen\"])\n", + "ds = nlmod.layers.set_model_top(ds, new_top)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# we then determine the part of each cell that is covered by sea from the original fine ahn\n", + "ds[\"sea\"] = nlmod.read.rws.calculate_sea_coverage(ahn, ds=ds, method=\"average\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -287,20 +337,12 @@ "metadata": {}, "outputs": [], "source": [ - "# voeg grote oppervlaktewaterlichamen toe o.b.v. shapefile\n", - "da_name = \"rws_oppwater\"\n", - "rws_ds = nlmod.read.rws.get_surface_water(\n", - " ds, da_name, cachedir=ds.cachedir, cachename=da_name\n", - ")\n", - "# add data to model dataset\n", - "ds.update(rws_ds)\n", - "\n", "# build ghb package\n", "ghb = nlmod.gwf.ghb(\n", " ds,\n", " gwf,\n", - " bhead=f\"{da_name}_stage\",\n", - " cond=f\"{da_name}_cond\",\n", + " bhead=0.0,\n", + " cond=ds[\"sea\"] * ds[\"area\"] / 1.0,\n", " auxiliary=18_000.0,\n", ")" ] @@ -328,13 +370,9 @@ "metadata": {}, "outputs": [], "source": [ - "# surface level drain\n", - "ahn_ds = nlmod.read.ahn.get_ahn(ds, cachedir=ds.cachedir, cachename=\"ahn\")\n", - "# add data to model dataset\n", - "ds.update(ahn_ds)\n", - "\n", "# build surface level drain package\n", - "drn = nlmod.gwf.surface_drain_from_ds(ds, gwf, resistance=10.0)" + "elev = ds[\"ahn\"].where(ds[\"sea\"] == 0)\n", + "drn = nlmod.gwf.surface_drain_from_ds(ds, gwf, elev=elev, resistance=10.0)" ] }, { @@ -356,7 +394,7 @@ "ds.update(knmi_ds)\n", "\n", "# create recharge package\n", - "rch = nlmod.gwf.rch(ds, gwf, mask=ds[\"rws_oppwater_cond\"] == 0)" + "rch = nlmod.gwf.rch(ds, gwf, mask=ds[\"sea\"] == 0)" ] }, { @@ -456,9 +494,7 @@ "pmv.plot_bc(\"GHB\", plotAll=True, alpha=0.1, label=\"GHB\")\n", "pmv.plot_bc(\"DRN\", plotAll=True, alpha=0.1, label=\"DRN\")\n", "# pmv.plot_bc(\"RCH\", plotAll=True, alpha=0.1, label=\"RCH\")\n", - "nlmod.plot.surface_water(\n", - " ds, ax=ax, hatch=\".\", edgecolor=\"k\", facecolor=\"none\", label=\"North Sea\"\n", - ")\n", + "municipalities.plot(edgecolor=\"k\", facecolor=\"none\", ax=ax)\n", "pmv.plot_grid(linewidth=0.25)\n", "ax.set_xlabel(\"x [km RD]\")\n", "ax.set_ylabel(\"y [km RD]\");" @@ -497,11 +533,13 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = nlmod.plot.get_map(extent_hbossche)\n", - "nlmod.plot.data_array(ctop.isel(time=-1), ds=ds, ax=ax, cmap=\"Spectral_r\")\n", - "nlmod.plot.surface_water(ds, ax=ax, hatch=\".\", edgecolor=\"k\", facecolor=\"none\")\n", - "ax.set_xlabel(\"x [km RD]\")\n", - "ax.set_ylabel(\"y [km RD]\");" + "ax = nlmod.plot.map_array(\n", + " ctop.isel(time=-1),\n", + " ds,\n", + " ilay=0,\n", + " cmap=\"Spectral_r\",\n", + ")\n", + "municipalities.plot(edgecolor=\"k\", facecolor=\"none\", ax=ax)" ] }, { @@ -536,7 +574,8 @@ " ax.set_ylim(bottom=-100)\n", " ax.set_xlabel(\"x [m]\")\n", " ax.set_ylabel(\"elevation [m NAP]\")\n", - " ax.set_title(f\"time = {c.time.isel(time=time_idx).values}\");" + " # convert to pandas timestamp for prettier printing\n", + " ax.set_title(f\"time = {pd.Timestamp(c.time.isel(time=time_idx).values)}\");" ] }, { @@ -554,13 +593,21 @@ "outputs": [], "source": [ "hf = nlmod.gwt.output.freshwater_head(ds, h, c)\n", - "hp = nlmod.gwt.output.pointwater_head(ds, hf, c)" + "hp = nlmod.gwt.output.pointwater_head(ds, hf, c)\n", + "\n", + "xr.testing.assert_allclose(h, hp)" ] } ], "metadata": { + "kernelspec": { + "display_name": "artesia", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "name": "python", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/docs/examples/17_unsaturated_zone_flow.ipynb b/docs/examples/17_unsaturated_zone_flow.ipynb new file mode 100644 index 00000000..68c675ad --- /dev/null +++ b/docs/examples/17_unsaturated_zone_flow.ipynb @@ -0,0 +1,427 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "71e9082c", + "metadata": {}, + "source": [ + "# Unsaturated zone flow\n", + "This notebook demonstrates the use of the Unsaturated Zone Flow (UZF) package in nlmod. The UZF-package can be used to simulate important processes in the unsaturated zone. These processes create a delay before precipitation reaches the saturated groundwater. In dry periods the water may even have evaporated before that. This notebook shows the difference in calculated head between a model with regular recharge (RCH) and evapotranspiration (EVT) packages, compared to a model with the UZF-package.\n", + "\n", + "We create a 1d model, of 1 row and 1 column, but with multiple layers, of a real location somewhere in the Netherlands. We use weather data from the KNMI as input for a transient simulation of 3 years with daily timetseps. This notebook can be used to vary the uzf-parameter, change the location (do not forget to alter the drain-elevation as well), or to play with the model-timestep." + ] + }, + { + "cell_type": "markdown", + "id": "0610edfd", + "metadata": {}, + "source": [ + "## Import packages and setup logging" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbf2d8ce", + "metadata": {}, + "outputs": [], + "source": [ + "# import packages\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import nlmod\n", + "\n", + "# set up pretty logging and show package versions\n", + "nlmod.util.get_color_logger()\n", + "nlmod.show_versions()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34f28281-fa68-4618-b063-8b9604306974", + "metadata": {}, + "outputs": [], + "source": [ + "# Ignore warning about 1d layers, in numpy 1.26.1 and flopy 3.4.3\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\", message=\"Conversion of an array with ndim > 0 to a scalar is deprecated, and will error in future.\")" + ] + }, + { + "cell_type": "markdown", + "id": "9813a7da", + "metadata": {}, + "source": [ + "## Generate a model Dataset\n", + "We first set the model_name and model workspace, which we will use later to write the model files, and so we can determine the cache-directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cc92261", + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"test_uzf\"\n", + "model_ws = os.path.join(\"data\", model_name)\n", + "figdir, cachedir = nlmod.util.get_model_dirs(model_ws)" + ] + }, + { + "cell_type": "markdown", + "id": "8de98855", + "metadata": {}, + "source": [ + "We define a location with a corresponding drainage elevation. From the location we calculate an extent of 100 by 100 meter, and download REGIS." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "341f7f9d", + "metadata": {}, + "outputs": [], + "source": [ + "x = 37_400\n", + "y = 412_600\n", + "drn_elev = 4.0\n", + "\n", + "# round x and y to 100, so we will download only one cell in regis\n", + "x = np.floor(x / 100) * 100\n", + "y = np.floor(y / 100) * 100\n", + "dx = dy = 100\n", + "extent = [x, x + dx, y, y + dy]\n", + "regis = nlmod.read.regis.get_regis(extent, cachename=\"regis.nc\", cachedir=cachedir)" + ] + }, + { + "cell_type": "markdown", + "id": "76deaeb7", + "metadata": {}, + "source": [ + "As the REGIS-data only contains one cell, we can visualize the properties of the layers in a pandas DataFrame. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbd30a5a", + "metadata": {}, + "outputs": [], + "source": [ + "regis.sel(x=regis.x[0], y=regis.y[0]).to_pandas()" + ] + }, + { + "cell_type": "markdown", + "id": "cdb20bc9", + "metadata": {}, + "source": [ + "As you can see, there are some NaN-values in the hydaulic conductivities (`kh` and `kv`). These will be filled when making a model Dataset, using fill-values and anisotropy values. See the info-messages after the commands below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71466503", + "metadata": {}, + "outputs": [], + "source": [ + "ds = nlmod.to_model_ds(\n", + " regis,\n", + " model_name=model_name,\n", + " model_ws=model_ws,\n", + " fill_value_kh=10.0,\n", + " fill_value_kv=10.0,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2e3b93b3", + "metadata": {}, + "source": [ + "We then add a time-dimension to our model Dataset and download knmi-data. We will have a calculation period of 3 year with daily timesteps, with a steady-state stress period with the weather of 2019 as a warmup period." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa5fb5c3", + "metadata": {}, + "outputs": [], + "source": [ + "time = pd.date_range(\"2020\", \"2023\", freq=\"D\")\n", + "ds = nlmod.time.set_ds_time(ds, start=\"2019\", time=time, steady_start=True)\n", + "\n", + "ds.update(\n", + " nlmod.read.knmi.get_recharge(\n", + " ds, method=\"separate\", cachename=\"recharge.nc\", cachedir=ds.cachedir\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "80994109", + "metadata": {}, + "source": [ + "## Generate a simulation (sim) and groundwater flow (gwf) object\n", + "We generate a model using with all basic packages. We add drainage level at 4.0 m NAP. As the top of our model is at 6.5 m NAP this will create an unsatuated zone of about 2.5 m." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b6b18d2", + "metadata": {}, + "outputs": [], + "source": [ + "# create simulation\n", + "sim = nlmod.sim.sim(ds)\n", + "\n", + "# create time discretisation\n", + "_ = nlmod.sim.tdis(ds, sim)\n", + "\n", + "# create groundwater flow model\n", + "gwf = nlmod.gwf.gwf(ds, sim, under_relaxation=True)\n", + "\n", + "# create ims\n", + "_ = nlmod.sim.ims(sim)\n", + "\n", + "# Create discretization\n", + "_ = nlmod.gwf.dis(ds, gwf)\n", + "\n", + "# create node property flow\n", + "_ = nlmod.gwf.npf(ds, gwf, icelltype=1)\n", + "\n", + "# creat storage\n", + "_ = nlmod.gwf.sto(ds, gwf, iconvert=1, sy=0.2, ss=1e-5)\n", + "\n", + "# Create the initial conditions package\n", + "_ = nlmod.gwf.ic(ds, gwf, starting_head=1.0)\n", + "\n", + "# Create the output control package\n", + "_ = nlmod.gwf.oc(ds, gwf)\n", + "\n", + "# set a drainage level with a resistance of 100 days in the layer that contains the drainage level\n", + "cond = ds[\"area\"] / 100.0\n", + "layer = nlmod.layers.get_layer_of_z(ds, drn_elev)\n", + "_ = nlmod.gwf.drn(ds, gwf, elev=drn_elev, cond=cond, layer=layer)" + ] + }, + { + "cell_type": "markdown", + "id": "62cbe11e", + "metadata": {}, + "source": [ + "## Run model with RCH and EVT packages\n", + "We first run the model with the Recharge (RCH) and Evapotranspiration (EVT) packages, as a reference for the model with the UZF-package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9546d7fc", + "metadata": {}, + "outputs": [], + "source": [ + "# create recharge package\n", + "nlmod.gwf.rch(ds, gwf)\n", + "\n", + "# create evapotranspiration package\n", + "nlmod.gwf.evt(ds, gwf);" + ] + }, + { + "cell_type": "markdown", + "id": "a3f051d0", + "metadata": {}, + "source": [ + "We run this model, read the heads and get the groundwater level (the head in the highest active cell). We save the groundwater level to a variable called `gwl_rch_evt`, which we will plot later on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a0939fb", + "metadata": {}, + "outputs": [], + "source": [ + "# run model\n", + "nlmod.sim.write_and_run(sim, ds, silent=True)\n", + "\n", + "# get heads\n", + "head_rch = nlmod.gwf.get_heads_da(ds)\n", + "\n", + "# calculate groundwater level\n", + "gwl_rch_evt = nlmod.gwf.output.get_gwl_from_wet_cells(head_rch, botm=ds[\"botm\"])" + ] + }, + { + "cell_type": "markdown", + "id": "34f1a8b6", + "metadata": {}, + "source": [ + "## Run model with UZF package\n", + "We then run the model again with the uzf-package. Before creating the uzf-package we remove the RCH- and EVT-packages.\n", + "\n", + "We choose some values for the residual water content, the saturated water content and the exponent used in the Brooks-Corey function. Other parameters are left to their defaults. The method `nlmod.gwf.uzf` will generate the UZF-package, using the variable `kv` from `ds` for the saturated vertical hydraulic conductivity, the variable `recharge` from `ds` for the infiltration rate and the variable `evaporation` from `ds` as the potential evapotranspiration rate.\n", + "\n", + "There can be multiple layers in the unsaturated zone, just like in the saturated zone. The method `nlmod.gwf.uzf` connects the unsaturated zone cells above each other.\n", + "\n", + "Because we want to plot the water content in the subsurface we will add some observations of the water concent to the uzf-package. We do this by adding the optional parameter `obs_z` to `nlmod.gwf.uzf`. This will create the observations in the corresponding uzf-cells, at the requested z-values (in m NAP)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52c711d6", + "metadata": {}, + "outputs": [], + "source": [ + "gwf.remove_package(\"RCH\")\n", + "gwf.remove_package(\"EVT\")\n", + "\n", + "# create uzf package\n", + "thtr = 0.1 # residual (irreducible) water content\n", + "thts = 0.3 # saturated water content\n", + "eps = 3.5 # exponent used in the Brooks-Corey function\n", + "# add observations of the water concent every 0.2 m, from 1 meter below the drainage-elevation to the model top\n", + "obs_z = np.arange(drn_elev - 1, ds[\"top\"].max(), 0.2)[::-1]\n", + "_ = nlmod.gwf.uzf(\n", + " ds,\n", + " gwf,\n", + " thtr=thtr,\n", + " thts=thts,\n", + " thti=thtr,\n", + " eps=eps,\n", + " print_input=True,\n", + " print_flows=True,\n", + " nwavesets=100, # Modflow 6 failed and advised to increase this value from the default value of 40\n", + " obs_z=obs_z,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b7835138", + "metadata": {}, + "source": [ + "We run the model again, and save the groundwater level to a variable called `gwl_uzf`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39f706a5", + "metadata": {}, + "outputs": [], + "source": [ + "# run model\n", + "nlmod.sim.write_and_run(sim, ds, silent=True)\n", + "\n", + "# get heads\n", + "head_rch = nlmod.gwf.get_heads_da(ds)\n", + "\n", + "# calculate groundwater level\n", + "gwl_uzf = nlmod.gwf.output.get_gwl_from_wet_cells(head_rch, botm=ds[\"botm\"])" + ] + }, + { + "cell_type": "markdown", + "id": "6938848c", + "metadata": {}, + "source": [ + "We read the water content from the observations, and use the name of the observations to determine the layer, row, columns and z-value of each observation, which we can use in our plot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af7128fd", + "metadata": {}, + "outputs": [], + "source": [ + "# %%\n", + "fname = os.path.join(ds.model_ws, f\"{ds.model_name}.uzf.obs.csv\")\n", + "obs = pd.read_csv(fname, index_col=0)\n", + "obs.index = pd.to_datetime(ds.time.start) + pd.to_timedelta(obs.index, \"D\")\n", + "kind, lay, row, col, z = zip(*[x.split(\"_\") for x in obs.columns])\n", + "lays = np.array([int(x) for x in lay])\n", + "rows = np.array([int(x) for x in row])\n", + "cols = np.array([int(x) for x in col])\n", + "z = np.array([float(x) for x in z])" + ] + }, + { + "cell_type": "markdown", + "id": "4d09726b", + "metadata": {}, + "source": [ + "## Compare models\n", + "We then make a plot to compare the heads in the two simulations we performed, and plot the water content we calculated in the UZF-calculation, and added observations for. We plot the water content in one vertical cell (the only) of the model. Figure layout thanks to Martin Vonk!\n", + "\n", + "The figure shows that the effect of precipitation on the groundwater level is less in summer if we also take the effect of the unsaturated zone into account (using UZF). In dry periods precipitation never reaches the groundwater level, as evaporation takes place before that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f25f53b9", + "metadata": {}, + "outputs": [], + "source": [ + "row = 0\n", + "col = 0\n", + "\n", + "f, ax = plt.subplot_mosaic(\n", + " [[\"P\"], [\"WC\"], [\"H\"]], figsize=(10, 8), height_ratios=[2, 5, 3], sharex=True\n", + ")\n", + "\n", + "# top ax\n", + "p = ds[\"recharge\"][:, row, col].to_pandas() * 1e3\n", + "p.plot(ax=ax[\"P\"], label=\"Precipitation\", color=\"C9\")\n", + "e = ds[\"evaporation\"][:, row, col].to_pandas() * 1e3\n", + "e.plot(ax=ax[\"P\"], label=\"Evaporation\", color=\"C8\")\n", + "ax[\"P\"].set_ylabel(\"[mm/d]\")\n", + "ax[\"P\"].set_ylim(bottom=0)\n", + "ax[\"P\"].legend(loc=(0, 1), ncol=2, frameon=False)\n", + "ax[\"P\"].grid()\n", + "\n", + "# middle ax\n", + "mask = (rows == row) & (cols == col)\n", + "XY = np.meshgrid(obs.index, z[mask])\n", + "theta = obs.loc[:, mask].transpose().values\n", + "pcm = ax[\"WC\"].pcolormesh(XY[0], XY[1], theta, cmap=\"viridis_r\", vmin=thtr, vmax=thts)\n", + "ax[\"WC\"].set_ylabel(\"z [m NAP]\")\n", + "ax[\"WC\"].grid()\n", + "# set xlim, as pcolormesh increases xlim a bit\n", + "ax[\"WC\"].set_xlim(ds.time[0], ds.time[-1])\n", + "nlmod.plot.colorbar_inside(pcm, ax=ax[\"WC\"], label=r\"Moisture Content $\\theta$ [-]\")\n", + "\n", + "# bottom ax\n", + "s = gwl_rch_evt[:, row, col].to_pandas()\n", + "ax[\"H\"].plot(s.index, s.values, color=\"C1\", linestyle=\"--\", label=\"GWL RCH+EVT\")\n", + "s = gwl_uzf[:, row, col].to_pandas()\n", + "ax[\"H\"].plot(s.index, s.values, color=\"C0\", linestyle=\"-\", label=\"GWL UZF\")\n", + "ax[\"H\"].set_ylabel(\"z [m NAP]\")\n", + "ax[\"H\"].legend(loc=(0, 0.98), ncol=2, frameon=False)\n", + "ax[\"H\"].grid()\n", + "\n", + "f.tight_layout(pad=0.8)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/getting_started.rst b/docs/getting_started.rst index dead56a5..95096449 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -123,6 +123,7 @@ On top of that there are some optional dependecies: - bottleneck (used in calculate_gxg) - geocube (used in add_min_ahn_to_gdf) - h5netcdf (used for the hdf5 backend of xarray) +- scikit-image (used in calculate_sea_coverage) These dependencies are only needed (and imported) in a single method or function. They can be installed using ``pip install nlmod[full]`` or ``pip install -e .[full]``. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 274aa18b..a1c41e05 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,20 +1,47 @@ nlmod ===== +nlmod is a Python package to build, run and visualize MODFLOW 6 groundwater models in the +Netherlands. The main focus is on building models in the Netherlands using publicly +available datasets, though lots of functionality could be applied to groundwater models +elsewhere. + .. image:: _static/logo_10000_2.png - :width: 400 + :width: 300 + :align: center :alt: The logo of nlmod -nlmod is a package with functions to pre-process, build and visualize MODFLOW -models. The main focus is on building models in the Netherlands using publicly -available datasets, though lots of functionality could be applied to any -groundwater model. - nlmod relies heavily on the wonderful features of `xarray `_ and `geopandas `_ for storing and manipulating data and `flopy `_ is used for building and running groundwater models. +We thank the following institutions for their contributions to the development of nlmod: + +.. image:: _static/logo_pwn.svg + :width: 200 + :alt: The logo of PWN + +.. image:: _static/logo_hhnk.svg + :width: 200 + :alt: The logo of Hoogheemraadschap Hollands Noorderkwartier + +.. image:: _static/logo_waternet.png + :width: 200 + :alt: The logo of Waternet + +.. image:: _static/logo_vitens.jpg + :width: 200 + :alt: The logo of Vitens + +.. image:: _static/logo_evides.png + :width: 200 + :alt: The logo of Evides + +.. image:: _static/logo_brabant_water.svg + :width: 200 + :alt: The logo of Brabant Water + Please note that the documentation for nlmod is still a work in progress, so bear with us if certain aspects are incomplete. @@ -30,4 +57,4 @@ bear with us if certain aspects are incomplete. Indices and tables ================== -* :ref:`genindex` \ No newline at end of file +* :ref:`genindex` diff --git a/nlmod/cache.py b/nlmod/cache.py index d84d4f16..e565988f 100644 --- a/nlmod/cache.py +++ b/nlmod/cache.py @@ -1,8 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Mon Nov 29 12:45:23 2021. - -@author: oebbe -""" import functools import importlib import inspect @@ -13,15 +8,17 @@ import dask import flopy +import joblib import numpy as np import pandas as pd import xarray as xr +from dask.diagnostics import ProgressBar logger = logging.getLogger(__name__) def clear_cache(cachedir): - """clears the cache in a given cache directory by removing all .pklz and + """Clears the cache in a given cache directory by removing all .pklz and corresponding .nc files. Parameters @@ -117,8 +114,16 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): # only use cache if the cache file and the pickled function arguments exist if os.path.exists(fname_cache) and os.path.exists(fname_pickle_cache): - with open(fname_pickle_cache, "rb") as f: - func_args_dic_cache = pickle.load(f) + # check if you can read the pickle, there are several reasons why a + # pickle can not be read. + try: + with open(fname_pickle_cache, "rb") as f: + func_args_dic_cache = pickle.load(f) + pickle_check = True + except (pickle.UnpicklingError, ModuleNotFoundError): + logger.info("could not read pickle, not using cache") + pickle_check = False + argument_check = False # check if the module where the function is defined was changed # after the cache was created @@ -131,18 +136,20 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): f"module of function {func.__name__} recently modified, not using cache" ) - cached_ds = xr.open_dataset(fname_cache, mask_and_scale=False) + cached_ds = xr.open_dataset(fname_cache) - # add netcdf hash to function arguments dic, see #66 - func_args_dic["_nc_hash"] = dask.base.tokenize(cached_ds) + if pickle_check: + # add netcdf hash to function arguments dic, see #66 + func_args_dic["_nc_hash"] = dask.base.tokenize(cached_ds) - # check if cache was created with same function arguments as - # function call - argument_check = _same_function_arguments( - func_args_dic, func_args_dic_cache - ) + # check if cache was created with same function arguments as + # function call + argument_check = _same_function_arguments( + func_args_dic, func_args_dic_cache + ) - if modification_check and argument_check: + cached_ds = _check_for_data_array(cached_ds) + if modification_check and argument_check and pickle_check: if dataset is None: logger.info(f"using cached data -> {cachename}") return cached_ds @@ -157,6 +164,10 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): result = func(*args, **kwargs) logger.info(f"caching data -> {cachename}") + if isinstance(result, xr.DataArray): + # set the DataArray as a variable in a new Dataset + result = xr.Dataset({"__xarray_dataarray_variable__": result}) + if isinstance(result, xr.Dataset): # close cached netcdf (otherwise it is impossible to overwrite) if os.path.exists(fname_cache): @@ -164,10 +175,21 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): cached_ds.close() # write netcdf cache - result.to_netcdf(fname_cache) + # check if dataset is chunked for writing with dask.delayed + first_data_var = list(result.data_vars.keys())[0] + if result[first_data_var].chunks: + delayed = result.to_netcdf(fname_cache, compute=False) + with ProgressBar(): + delayed.compute() + # close and reopen dataset to ensure data is read from + # disk, and not from opendap + result.close() + result = xr.open_dataset(fname_cache, chunks="auto") + else: + result.to_netcdf(fname_cache) # add netcdf hash to function arguments dic, see #66 - temp = xr.open_dataset(fname_cache, mask_and_scale=False) + temp = xr.open_dataset(fname_cache) func_args_dic["_nc_hash"] = dask.base.tokenize(temp) temp.close() @@ -176,14 +198,128 @@ def decorator(*args, cachedir=None, cachename=None, **kwargs): pickle.dump(func_args_dic, fpklz) else: raise TypeError(f"expected xarray Dataset, got {type(result)} instead") + result = _check_for_data_array(result) + return result + + return decorator + + +def cache_pickle(func): + """decorator to read/write the result of a function from/to a file to speed + up function calls with the same arguments. Should only be applied to + functions that: + + - return a picklable object + - have functions arguments of types that can be checked using the + _is_valid_cache functions + + 1. The directory and filename of the cache should be defined by the person + calling a function with this decorator. If not defined no cache is + created nor used. + 2. Create a new cached file if it is impossible to check if the function + arguments used to create the cached file are the same as the current + function arguments. This can happen if one of the function arguments has a + type that cannot be checked using the _is_valid_cache function. + 3. Function arguments are pickled together with the cache to check later + if the cache is valid. + 4. This function uses `functools.wraps` and some home made + magic in _update_docstring_and_signature to add arguments of the decorator + to the decorated function. This assumes that the decorated function has a + docstring with a "Returns" heading. If this is not the case an error is + raised when trying to decorate the function. + """ + + # add cachedir and cachename to docstring + _update_docstring_and_signature(func) + + @functools.wraps(func) + def decorator(*args, cachedir=None, cachename=None, **kwargs): + # 1 check if cachedir and name are provided + if cachedir is None or cachename is None: + return func(*args, **kwargs) + + if not cachename.endswith(".pklz"): + cachename += ".pklz" + + fname_cache = os.path.join(cachedir, cachename) # pklz file + fname_pickle_cache = fname_cache.replace(".pklz", "__cache__.pklz") + + # create dictionary with function arguments + func_args_dic = {f"arg{i}": args[i] for i in range(len(args))} + func_args_dic.update(kwargs) + + # only use cache if the cache file and the pickled function arguments exist + if os.path.exists(fname_cache) and os.path.exists(fname_pickle_cache): + # check if you can read the function argument pickle, there are + # several reasons why a pickle can not be read. + try: + with open(fname_pickle_cache, "rb") as f: + func_args_dic_cache = pickle.load(f) + pickle_check = True + except (pickle.UnpicklingError, ModuleNotFoundError): + logger.info("could not read pickle, not using cache") + pickle_check = False + argument_check = False + + # check if the module where the function is defined was changed + # after the cache was created + time_mod_func = _get_modification_time(func) + time_mod_cache = os.path.getmtime(fname_cache) + modification_check = time_mod_cache > time_mod_func + + if not modification_check: + logger.info( + f"module of function {func.__name__} recently modified, not using cache" + ) + + # check if you can read the cached pickle, there are + # several reasons why a pickle can not be read. + try: + with open(fname_cache, "rb") as f: + cached_pklz = pickle.load(f) + except (pickle.UnpicklingError, ModuleNotFoundError): + logger.info("could not read pickle, not using cache") + pickle_check = False + + if pickle_check: + # add dataframe hash to function arguments dic + func_args_dic["_pklz_hash"] = joblib.hash(cached_pklz) + + # check if cache was created with same function arguments as + # function call + argument_check = _same_function_arguments( + func_args_dic, func_args_dic_cache + ) + + if modification_check and argument_check and pickle_check: + logger.info(f"using cached data -> {cachename}") + return cached_pklz + + # create cache + result = func(*args, **kwargs) + logger.info(f"caching data -> {cachename}") + + if isinstance(result, pd.DataFrame): + # write pklz cache + result.to_pickle(fname_cache) + + # add dataframe hash to function arguments dic + with open(fname_cache, "rb") as f: + temp = pickle.load(f) + func_args_dic["_pklz_hash"] = joblib.hash(temp) + # pickle function arguments + with open(fname_pickle_cache, "wb") as fpklz: + pickle.dump(func_args_dic, fpklz) + else: + raise TypeError(f"expected DataFrame, got {type(result)} instead") return result return decorator def _check_ds(ds, ds2): - """check if two datasets have the same dimensions and coordinates. + """Check if two datasets have the same dimensions and coordinates. Parameters ---------- @@ -199,12 +335,6 @@ def _check_ds(ds, ds2): True if the two datasets have the same grid and time discretization. """ - # first remove _FillValue from all coordinates - for coord in ds2.coords: - if coord in ds.coords: - if "_FillValue" in ds2[coord].attrs and "_FillValue" not in ds[coord].attrs: - del ds2[coord].attrs["_FillValue"] - for coord in ds2.coords: if coord in ds.coords: try: @@ -308,7 +438,7 @@ def _same_function_arguments(func_args_dic, func_args_dic_cache): def _get_modification_time(func): - """return the modification time of the module where func is defined. + """Return the modification time of the module where func is defined. Parameters ---------- @@ -330,10 +460,12 @@ def _get_modification_time(func): def _update_docstring_and_signature(func): - """add function arguments 'cachedir' and 'cachename' to the docstring and - signature of a function. The function arguments are added before the - "Returns" header in the docstring. If the function has no Returns header in - the docstring, the function arguments are not added to the docstring. + """Add function arguments 'cachedir' and 'cachename' to the docstring and signature + of a function. + + The function arguments are added before the "Returns" header in the + docstring. If the function has no Returns header in the docstring, the function + arguments are not added to the docstring. Parameters ---------- @@ -342,11 +474,16 @@ def _update_docstring_and_signature(func): Returns ------- - None. + None """ # add cachedir and cachename to signature sig = inspect.signature(func) cur_param = tuple(sig.parameters.values()) + if cur_param[-1].name == "kwargs": + add_kwargs = cur_param[-1] + cur_param = cur_param[:-1] + else: + add_kwargs = None new_param = cur_param + ( inspect.Parameter( "cachedir", inspect.Parameter.POSITIONAL_OR_KEYWORD, default=None @@ -355,6 +492,8 @@ def _update_docstring_and_signature(func): "cachename", inspect.Parameter.POSITIONAL_OR_KEYWORD, default=None ), ) + if add_kwargs is not None: + new_param = new_param + (add_kwargs,) sig = sig.replace(parameters=new_param) func.__signature__ = sig @@ -379,3 +518,37 @@ def _update_docstring_and_signature(func): new_doc = "".join((mod_before, after)) func.__doc__ = new_doc return + + +def _check_for_data_array(ds): + """ + Check if the saved NetCDF-file represents a DataArray or a Dataset, and return this + data-variable. + + The file contains a DataArray when a variable called "__xarray_dataarray_variable__" + is present in the Dataset. If so, return a DataArray, otherwise return the Dataset. + + By saving the DataArray, the coordinate "spatial_ref" was saved as a separate + variable. Therefore, add this variable as a coordinate to the DataArray again. + + Parameters + ---------- + ds : xr.Dataset + Dataset with dimensions and coordinates. + + Returns + ------- + ds : xr.Dataset or xr.DataArray + A Dataset or DataArray containing the cached data. + + """ + if "__xarray_dataarray_variable__" in ds: + if "spatial_ref" in ds: + spatial_ref = ds.spatial_ref + else: + spatial_ref = None + # the method returns a DataArray, so we return only this DataArray + ds = ds["__xarray_dataarray_variable__"] + if spatial_ref is not None: + ds = ds.assign_coords({"spatial_ref": spatial_ref}) + return ds diff --git a/nlmod/data/geotop/Lithoklasse.voleg b/nlmod/data/geotop/Lithoklasse.voleg new file mode 100644 index 00000000..ddd47fde --- /dev/null +++ b/nlmod/data/geotop/Lithoklasse.voleg @@ -0,0 +1,8 @@ +1 organische stof (veen) 157 78 64 255 organische stof (veen) +2 klei 0 146 0 255 klei +3 kleiig zand, zandige klei en leem 194 207 92 255 kleiig zand, zandige klei en leem +5 zand fijn 255 255 0 255 zand fijn +6 zand midden 243 225 6 255 zand midden +7 zand grof 231 195 22 255 zand grof +8 grind 216 163 32 255 grind +9 schelpen 95 95 255 255 schelpen diff --git a/nlmod/data/geotop/Lithostratigrafie.voleg b/nlmod/data/geotop/Lithostratigrafie.voleg new file mode 100644 index 00000000..49d5f50a --- /dev/null +++ b/nlmod/data/geotop/Lithostratigrafie.voleg @@ -0,0 +1,17 @@ +1 AAOP 200 200 200 255 AAOP +8 ANAWA 118 147 60 255 ANAWA +10 BNAWA 102 205 171 255 BNAWA +14 NAWA 152 220 117 255 NAWA +17 NIHO 208 130 40 255 NIHO +24 NAWO 63 165 76 255 NAWO +28 NIBA 152 47 10 255 NIBA +35 BXWIKO 255 255 80 255 BXWIKO +44 BX 255 235 0 255 BX +54 KW 172 169 43 255 KW +56 EE 210 255 115 255 EE +71 PZWA 255 204 0 255 PZWA +74 OO 118 157 39 255 OO +77 BR 108 188 150 255 BR +79 RU 184 123 238 255 RU +82 TO 90 159 219 255 TO +85 DO 216 191 216 255 DO diff --git a/nlmod/data/geotop/geo_eenheden.csv b/nlmod/data/geotop/geo_eenheden.csv index 55d2b138..4a114bf8 100644 --- a/nlmod/data/geotop/geo_eenheden.csv +++ b/nlmod/data/geotop/geo_eenheden.csv @@ -12,6 +12,7 @@ verbreiding van de Formatie van Nieuwkoop, Hollandveen Laagpakket" 1070,OEC,"Formatie van Echteld, gelegen boven de Formatie van Nieuwkoop, Hollandveen Laagpakket" 1080,NAWOBE,"Formatie van Naaldwijk, Laagpakket van Wormer, Laag van Bergen" 1090,NIHO,"Formatie van Nieuwkoop, Hollandveen Laagpakket" +1095,NINB,"Formatie van Nieuwkoop, Laagpakket van Nij Beets" 1100,NAWO,"Formatie van Naaldwijk, Laagpakket van Wormer" 1110,NWNZ,"Formatie van Naaldwijk, Laagpakketten van Wormer en Zandvoort (gecombineerde eenheid in modelgebied Zeeland)" diff --git a/nlmod/data/geotop/geotop.gleg b/nlmod/data/geotop/geotop.gleg new file mode 100644 index 00000000..c98c7c1c --- /dev/null +++ b/nlmod/data/geotop/geotop.gleg @@ -0,0 +1,130 @@ +1 organische stof (veen) 157 78 64 255 x +10 zand 243 225 6 255 x +10-zand zand 243 225 6 255 x +11 kalksteen 140 180 255 255 x +11-kalksteen kalksteen 140 180 255 255 x +1-veen organische stof (veen) 157 78 64 255 x +2 klei 0 146 0 255 x +2-klei klei 0 146 0 255 x +3 "kleiig zand, zandige klei en leem" 194 207 92 255 x +3-kleiig zand "kleiig zand, zandige klei en leem" 194 207 92 255 x +5 zand fijn 255 255 0 255 x +5-zand fijn zand fijn 255 255 0 255 x +6 zand midden 243 225 6 255 x +6-zand midden zand midden 243 225 6 255 x +7 zand grof 231 195 22 255 x +7-zand grof zand grof 231 195 22 255 x +8 grind 216 163 32 255 x +8-grind grind 216 163 32 255 x +9 schelpen 95 95 255 255 x +9-schelpen schelpen 95 95 255 255 x +AA Antropogeen 200 200 200 255 x +AAES Antropogeen esdekken 110 110 110 255 x +AAOP Antropogeen opgebrachte grond 200 200 200 255 x +ABEOM "Stroombaan generatie A, Formatie van Beegden, Laagpakket van Oost-Maarland" 118 147 60 255 x +AEC "Stroombaan generatie A, Formatie van Echteld" 118 147 60 255 x +AK Formatie van Aken 152 231 205 255 x +ANAWA "Stroombaan generatie A, Formatie van Naaldwijk, Laagpakket van Walcheren" 118 147 60 255 x +ANAWO "Stroombaan generatie A, Formatie van Naaldwijk, Laagpakket van Wormer" 118 147 60 255 x +AP Formatie van Appelscha 218 165 32 255 x +BE Formatie van Beegden 200 200 255 255 x +BEC "Stroombaan generatie B, Formatie van Echteld" 102 205 171 255 x +BEOM "Formatie van Beegden, Laagpakket van Oost-Maarland" 129 105 123 255 x +BERO "Formatie van Beegden, Laagpakket van Rosmalen" 160 140 155 255 x +BNAWA "Stroombaan generatie B, Formatie van Naaldwijk, Laagpakket van Walcheren" 102 205 171 255 x +BNAWO "Stroombaan generatie B, Formatie van Naaldwijk, Laagpakket van Wormer" 102 205 171 255 x +BR Formatie van Breda 108 188 150 255 x +BX Formatie van Boxtel 255 235 0 255 x +BXAS "Formatie van Boxtel, Eem afzettingen" 127 63 63 255 x +BXBS "Formatie van Boxtel, Laagpakket van Best" 235 205 0 255 x +BXDE "Formatie van Boxtel, Laagpakket van Delwijnen" 225 225 130 255 x +BXDEKO "Formatie van Boxtel, laagpakketten van Delwijnen en Kootwijk" 225 225 130 255 x +BXKO "Formatie van Boxtel, Laagpakket van Kootwijk" 255 255 190 255 x +BXLM "Formatie van Boxtel, Laagpakket van Liempde" 225 190 0 255 x +BXLEEM "Formatie van Boxtel, Eerste Leemlaag" 220 220 220 255 x +BXSC "Formatie van Boxtel, Laagpakket van Schimmert" 255 205 0 255 x +BXSI1 "Formatie van Boxtel, Laagpakket van Singraven (bovenste deel)" 190 190 0 255 x +BXWI "Formatie van Boxtel, Laagpakket van Wierden" 255 255 80 255 x +BXSI2 "Formatie van Boxtel, Laagpakket van Singraven (onderste deel)" 255 255 130 255 x +BXWIKO "Formatie van Boxtel, laagpakketten van Wierden en Kootwijk" 255 255 80 255 x +BXWISIKO "Formatie van Boxtel, Laagpakketten van Wierden, Singraven en Kootwijk" 255 255 80 255 x +CEC "Stroombaan generatie C, Formatie van Echteld" 170 196 255 255 x +CNAWA "Stroombaan generatie C, Formatie van Naaldwijk, Laagpakket van Walcheren" 170 196 255 255 x +CNAWO "Stroombaan generatie C, Formatie van Naaldwijk, Laagpakket van Wormer" 170 196 255 255 x +DEC "Stroombaan generatie D, Formatie van Echteld" 102 153 205 255 x +DN Formatie van Drachten 250 250 210 255 x +DNAWA "Stroombaan generatie D, Formatie van Naaldwijk, Laagpakket van Walcheren" 102 153 205 255 x +DNAWO "Stroombaan generatie D, Formatie van Naaldwijk, Laagpakket van Wormer" 102 153 205 255 x +DO Formatie van Dongen 216 191 216 255 x +DOAS "Formatie van Dongen, Laagpakket van Asse" 186 146 141 255 x +DOIE "Formatie van Dongen, Laagpakket van Ieper" 206 176 191 255 x +DR Formatie van Drente 255 127 80 255 x +DRGI "Formatie van Drente, Laagpakket van Gieten" 235 97 30 255 x +DRSC "Formatie van Drente, Laagpakket van Schaarsbergen" 255 127 80 255 x +DRUI "Formatie van Drente, Laagpakket van Uitdam" 225 82 5 255 x +DRUIOO "Formatie van Drente, Laagpakket van Uitdam, Laag van Oosterdok" 225 82 5 255 +EC Formatie van Echteld 170 255 245 255 x +EE Eem Formatie 210 255 115 255 x +EEC "Stroombaan generatie E, Formatie van Echteld" 27 101 175 255 x +EEWB Eem-Woudenberg Formatie 190 255 115 255 x +ENAWA "Stroombaan generatie E, Formatie van Naaldwijk, Laagpakket van Walcheren" 27 101 175 255 x +ENAWO "Stroombaan generatie E, Formatie van Naaldwijk, Laagpakket van Wormer" 27 101 175 255 x +GE Gestuwde afzettingen 156 156 156 255 x +GU Formatie van Gulpen 245 222 179 255 x +HL Holoceen 12 129 12 255 x +HO Formatie van Houthem 210 105 30 255 x +HT Formatie van Heijenrath 178 34 34 255 x +IE Formatie van Inden 236 121 193 255 x +KI Kiezelooliet Formatie 188 143 143 255 x +KR "Formatie van Kreftenheye, incl. Laagpakket van Delwijnen" 176 48 96 255 x +KK1 "Kreekrak Formatie (boven NIHO)" 68 138 112 255 x +KK2 "Kreekrak Formatie (onder NIHO)" 68 138 112 255 x +KRBXDE "Formatie van Kreftenheye, incl. Laagpakket van Delwijnen" 176 48 96 255 x +KRTW "Formatie van Kreftenheye, Laagpakket van Twello" 156 18 46 255 x +KRWE "Formatie van Kreftenheye, Laagpakket van Well" 176 48 96 255 x +KRWY "Formatie van Kreftenheye, Laag van Wijchen" 86 0 0 255 x +KRZU "Formatie van Kreftenheye, Laagpakket van Zutphen" 136 0 0 255 x +KW Formatie van Koewacht 172 169 43 255 x +KW1 Formatie van Koewacht (kleiige top) 152 139 0 255 x +LA Formatie van Landen 208 32 144 255 x +MS Formatie van Maassluis 135 206 235 255 x +MT Formatie van Maastricht 255 160 102 255 x +MV Maaiveld 255 255 200 255 x +NA Formatie van Naaldwijk 105 200 100 255 x +NASC "Formatie van Naaldwijk, Laagpakket van Schoorl" 255 255 160 255 x +NAWA "Formatie van Naaldwijk, Laagpakket van Walcheren" 152 220 117 255 x +NAWO "Formatie van Naaldwijk, Laagpakket van Wormer" 63 165 76 255 x +NAWOBE "Formatie van Naaldwijk, Laagpakket van Wormer, Laag van Bergen" 0 157 155 255 x +NAWOVE "Formatie van Naaldwijk, Laagpakket van Wormer, Laag van Velsen" 189 183 107 255 x +NAZA "Formatie van Naaldwijk, Laagpakket van Zandvoort (boven NIHO)" 254 183 56 255 x +NAZA2 "Formatie van Naaldwijk, Laagpakket van Zandvoort (onder NIHO)" 254 183 56 255 x +NI Formatie van Nieuwkoop 208 130 40 255 x +NIBA "Formatie van Nieuwkoop, Basisveen Laag" 152 47 10 255 x +NIGR "Formatie van Nieuwkoop, Laagpakket van Griendtsveen" 132 88 44 255 x +NIHO "Formatie van Nieuwkoop, Hollandveen Laagpakket" 208 130 40 255 x +NINB "Formatie van Nieuwkoop, Laagpakket van Nij Beets" 180 90 20 255 x +OEC "Formatie van Echteld, gelegen boven Hollandveen Laagpakket" 170 255 245 255 x +ONAWA "Formatie van Naaldwijk, Laagpakket van Walcheren, boven Laagpakket van Zandvoort" 152 220 117 255 x +OO Formatie van Oosterhout 118 157 39 255 x +PE Formatie van Peelo 238 130 238 255 x +PL Pleistoceen 255 255 150 255 x +PZ Formatie van Peize 255 255 0 255 x +PZBA "Formatie van Peize, Laagpakket van Balk" 255 255 0 255 x +PZWA Formaties van Peize en Waalre 255 204 0 255 x +RU Rupel Formatie 184 123 238 255 x +RUBO "Rupel Formatie, Laagpakket van Boom" 154 78 163 255 x +ST Formatie van Sterksel 205 92 92 255 x +SY Formatie van Stramproy 255 228 181 255 x +TE Tertiair 253 217 65 255 x +TO Formatie van Tongeren 90 159 219 255 x +TOGO "Formatie van Tongeren, Laagpakket van Goudsberg" 70 129 169 255 x +TOZE "Formatie van Tongeren, Laagpakket van Zelzate" 90 159 219 255 x +TOZEBA "Formatie van Tongeren, Laagpakket van Zelzate, Laag van Bassevelde" 90 159 219 255 x +TOZERU "Formatie van Tongeren, Laagpakket van Zelzate, Laag van Ruisbroek" 90 159 219 255 x +TOZEWA "Formatie van Tongeren, Laagpakket van Zelzate, Laag van Watervliet" 80 144 194 255 x +UR Formatie van Urk 189 183 107 255 x +URTY "Formatie van Urk, Laagpakket van Tijnje" 169 163 87 255 x +VA Formatie van Vaals 21 153 79 255 x +VE Formatie van Veldhoven 102 100 16 255 x +WA Formatie van Waalre 255 165 0 255 x +WB Formatie van Woudenberg 137 67 30 255 x diff --git a/nlmod/dims/__init__.py b/nlmod/dims/__init__.py index 5cbdf702..d5892790 100644 --- a/nlmod/dims/__init__.py +++ b/nlmod/dims/__init__.py @@ -1,4 +1,5 @@ from . import base, grid, layers, resample, time +from .attributes_encodings import * from .base import * from .grid import * from .layers import * diff --git a/nlmod/dims/attributes_encodings.py b/nlmod/dims/attributes_encodings.py new file mode 100644 index 00000000..13512342 --- /dev/null +++ b/nlmod/dims/attributes_encodings.py @@ -0,0 +1,276 @@ +import numpy as np + +dim_attrs = { + "time": { + "name": "Time", + "description": "End time of the stress period", + }, + "botm": { + "name": "Bottom elevation", + "description": "Bottom elevation for each model cell", + "units": "mNAP", + "valid_min": -2000.0, + "valid_max": 500.0, + }, + "top": { + "name": "Top elevation", + "description": "Top elevation for each model cell", + "units": "mNAP", + "valid_min": -2000.0, + "valid_max": 500.0, + }, + "kh": { + "name": "Horizontal hydraulic conductivity", + "description": "Horizontal hydraulic conductivity for each model cell", + "units": "m/day", + "valid_min": 0.0, + "valid_max": 1000.0, + }, + "kv": { + "name": "Vertical hydraulic conductivity", + "description": "Vertical hydraulic conductivity for each model cell", + "units": "m/day", + "valid_min": 0.0, + "valid_max": 1000.0, + }, + "ss": { + "name": "Specific storage", + "description": "Specific storage for each model cell", + "units": "1/m", + "valid_min": 0.0, + "valid_max": 1.0, + }, + "sy": { + "name": "Specific yield", + "description": "Specific yield for each model cell", + "units": "-", + "valid_min": 0.0, + "valid_max": 1.0, + }, + "porosity": { + "name": "Porosity", + "description": "Porosity for each model cell", + "units": "-", + "valid_min": 0.0, + "valid_max": 1.0, + }, + "recharge": { + "name": "Recharge", + "description": "Recharge for each model cell", + "units": "m/day", + "valid_min": 0.0, + "valid_max": 0.5, + }, + "heads": { + "name": "Heads", + "description": "Point water heads for each model cell", + "units": "mNAP", + "valid_min": -100.0, + "valid_max": 500.0, + }, + "starting_head": { + "name": "Starting head", + "description": "Starting head for each model cell", + "units": "mNAP", + "valid_min": -100.0, + "valid_max": 500.0, + }, + "freshwater_head": { + "name": "Freshwater head", + "description": "Freshwater head for each model cell", + "units": "mNAP", + "valid_min": -100.0, + "valid_max": 500.0, + }, + "pointwater_head": { + "name": "Pointwater head", + "description": "Pointwater head for each model cell", + "units": "mNAP", + "valid_min": -100.0, + "valid_max": 500.0, + }, + "density": { + "name": "Density", + "description": "Density for each model cell", + "units": "kg/m3", + "valid_min": 950.0, + "valid_max": 1200.0, + }, + "area": { + "name": "Cell area", + "description": "Cell area for each model cell", + "units": "m2", + "valid_min": 0.0, + "valid_max": 1e8, + }, +} + + +encoding_requirements = { + "heads": {"dval_max": 0.005}, + "botm": {"dval_max": 0.005}, + "top": {"dval_max": 0.005}, + "kh": {"dval_max": 1e-6}, + "kv": {"dval_max": 1e-6}, + "ss": {"dval_max": 1e-8}, + "sy": {"dval_max": 0.005}, + "porosity": {"dval_max": 0.005}, + "recharge": {"dval_max": 0.0005}, + "starting_head": {"dval_max": 0.005}, + "freshwater_head": {"dval_max": 0.005}, + "pointwater_head": {"dval_max": 0.005}, + "density": {"dval_max": 0.005}, + "area": {"dval_max": 0.05}, +} + + +def get_encodings( + ds, set_encoding_inplace=True, allowed_to_read_data_vars_for_minmax=True +): + """Get the encoding for the data_vars. Based on the minimum values and maximum + values set in `dim_attrs` and the maximum allowed difference from + `encoding_requirements`. + + If a loss of data resolution is allowed floats can also be stored at int16, halfing + the space required for storage. The maximum acceptabel loss in resolution + (`dval_max`) is compared with the expected loss in resolution + (`is_int16_allowed()`). + + If `set_encoding_inplace` is False, a dictionary with encodings is returned that + can be passed as argument to `ds.to_netcdf()`. If True, the encodings are set + inplace; they are stored in the `ds["var"].encoding` for each var seperate. + + If encoding is specified as argument in `ds.to_netcdf()` the encoding stored in the + `ds["var"].encoding` for each var is ignored. + + Parameters + ---------- + ds : xarray.Dataset + Dataset containing the data_vars + set_encoding_inplace : bool + Set the encoding inplace, by default True + allowed_to_data_vars : bool + If True, only data_vars that are allowed to be read are used to calculate the + minimum and maximum values to estimate the effect of precision loss. + If False, min max from dim_attrs are used. By default True. + + Returns + ------- + encodings : dict or None + Dictionary containing the encodings for each data_var + + TODO: add support for strings + """ + encodings = {} + for varname, da in ds.data_vars.items(): + # Encoding for strings is not supported by netCDF + if np.issubdtype(da.dtype, np.character): + continue + + assert ( + "_FillValue" not in da.attrs + ), f"Custom fillvalues are not supported. {varname} has a fillvalue set." + + encoding = { + "zlib": True, + "complevel": 5, + "fletcher32": True, # Store checksums to detect corruption + } + + isfloat = np.issubdtype(da.dtype, np.floating) + isint = np.issubdtype(da.dtype, np.integer) + + # set the dtype, scale_factor and add_offset + if isfloat and varname in encoding_requirements and varname in dim_attrs: + dval_max = encoding_requirements[varname]["dval_max"] + + if allowed_to_read_data_vars_for_minmax: + vmin = float(da.min()) + vmax = float(da.max()) + else: + vmin = dim_attrs[varname]["valid_min"] + vmax = dim_attrs[varname]["valid_max"] + + float_as_int16 = is_int16_allowed(vmin, vmax, dval_max) + + if float_as_int16: + scale_factor, add_offset = compute_scale_and_offset(vmin, vmax) + encoding["dtype"] = "int16" + encoding["scale_factor"] = scale_factor + encoding["add_offset"] = add_offset + encoding["_FillValue"] = -32767 # default for NC_SHORT + # result = (np.array([vmin, vmax]) - add_offset) / scale_factor + else: + encoding["dtype"] = "float32" + + elif isint and allowed_to_read_data_vars_for_minmax: + vmin = int(da.min()) + vmax = int(da.max()) + + if vmin >= -32766 and vmax <= 32767: + encoding["dtype"] = "int16" + elif vmin >= -2147483646 and vmax <= 2147483647: + encoding["dtype"] = "int32" + else: + encoding["dtype"] = "int64" + else: + pass + + if set_encoding_inplace: + da.encoding = encoding + else: + encodings[varname] = encoding + + if set_encoding_inplace: + return None + else: + return encodings + + +def compute_scale_and_offset(min_value, max_value): + """ + Computes the scale_factor and offset for the dataset using a min_value and max_value, + and int16. Useful for maximizing the compression of a dataset. + + Parameters + ---------- + min_value : float + Minimum value of the dataset + max_value : float + Maximum value of the dataset + + Returns + ------- + scale_factor : float + Scale factor for the dataset + add_offset : float + Add offset for the dataset + """ + # stretch/compress data to the available packed range + # from -32766 to 32767, because -32767 is the default fillvalue. + width = 32766 + 32767 + scale_factor = (max_value - min_value) / width + add_offset = 1.0 + scale_factor * (width - 1) / 2 + return scale_factor, add_offset + + +def is_int16_allowed(vmin, vmax, dval_max): + """compute the loss of resolution by storing a float as int16 (`dval`). And + compare it with the maximum allowed loss of resolution (`dval_max`). + + Parameters + ---------- + vmin : float + Minimum value of the dataset + vmax : float + Maximum value of the dataset + dval_max : float + Maximum allowed loss of resolution + + Returns + ------- + bool + True if the loss of resolution is allowed, False otherwise""" + nsteps = 32766 + 32767 + dval = (vmax - vmin) / nsteps + return dval <= dval_max diff --git a/nlmod/dims/base.py b/nlmod/dims/base.py index b4e62e86..5c238db9 100644 --- a/nlmod/dims/base.py +++ b/nlmod/dims/base.py @@ -7,8 +7,9 @@ from scipy.spatial import cKDTree from .. import util +from ..epsg28992 import EPSG_28992 from . import resample -from .layers import fill_nan_top_botm_kh_kv, set_idomain +from .layers import fill_nan_top_botm_kh_kv logger = logging.getLogger(__name__) @@ -42,7 +43,7 @@ def set_ds_attrs(ds, model_name, model_ws, mfversion="mf6", exe_name=None): ds.attrs["model_name"] = model_name ds.attrs["mfversion"] = mfversion fmt = "%Y%m%d_%H:%M:%S" - ds.attrs["model_dataset_created_on"] = dt.datetime.now().strftime(fmt) + ds.attrs["created_on"] = dt.datetime.now().strftime(fmt) if exe_name is None: exe_name = util.get_exe_path(mfversion) @@ -166,7 +167,7 @@ def to_model_ds( ds = set_ds_attrs(ds, model_name, model_ws) ds.attrs["transport"] = int(transport) - # fill nan's and add idomain + # fill nan's if fill_nan: ds = fill_nan_top_botm_kh_kv( ds, @@ -174,8 +175,6 @@ def to_model_ds( fill_value_kh=fill_value_kh, fill_value_kv=fill_value_kv, ) - else: - ds = set_idomain(ds, remove_nan_layers=False) return ds @@ -207,7 +206,7 @@ def extrapolate_ds(ds, mask=None): # all of the model cells are is inside the known area return ds if mask.all(): - raise (Exception("The model only contains NaNs")) + raise (ValueError("The model only contains NaNs")) if "gridtype" in ds.attrs and ds.gridtype == "vertex": x = ds.x.data y = ds.y.data @@ -225,12 +224,18 @@ def extrapolate_ds(ds, mask=None): continue data = ds[key].data if ds[key].dims == dims: - data[mask] = data[~mask][i] + if np.isnan(data[mask]).sum() > 0: # do not update if no NaNs + data[mask] = data[~mask, i] elif ds[key].dims == ("layer",) + dims: for lay in range(len(ds["layer"])): - data[lay][mask] = data[lay][~mask][i] + if np.isnan(data[lay, mask]).sum() > 0: # do not update if no NaNs + data[lay, mask] = data[lay, ~mask][i] else: - raise (Exception(f"Dimensions {ds[key].dims} not supported")) + logger.warning( + f"Data variable '{key}' not extrapolated because " + f"dimensions are not {dims}." + ) + # raise (Exception(f"Dimensions {ds[key].dims} not supported")) # make sure to set the data (which for some reason is sometimes needed) ds[key].data = data return ds @@ -446,10 +451,10 @@ def _get_vertex_grid_ds( coords = {"layer": layers, "y": y, "x": x} dims = ("layer", "icell2d") ds = xr.Dataset( - data_vars=dict( - top=(dims[1:], top), - botm=(dims, botm), - ), + data_vars={ + "top": (dims[1:], top), + "botm": (dims, botm), + }, coords=coords, attrs=attrs, ) @@ -466,7 +471,7 @@ def _get_vertex_grid_ds( for i in range(ncpl): icvert[i, : cell2d[i][3]] = cell2d[i][4:] ds["icvert"] = ("icell2d", "icv"), icvert - ds["icvert"].attrs["_FillValue"] = nodata + ds["icvert"].attrs["nodata"] = nodata if crs is not None: ds.rio.set_crs(crs) @@ -484,7 +489,7 @@ def get_ds( botm=None, kh=10.0, kv=1.0, - crs=28992, + crs=EPSG_28992, xorigin=0.0, yorigin=0.0, angrot=0.0, @@ -599,7 +604,7 @@ def get_ds( resample._set_angrot_attributes(extent, xorigin, yorigin, angrot, attrs) x, y = resample.get_xy_mid_structured(attrs["extent"], delr, delc) - coords = dict(x=x, y=y, layer=layer) + coords = {"x": x, "y": y, "layer": layer} if angrot != 0.0: affine = resample.get_affine_mod_to_world(attrs) xc, yc = affine * np.meshgrid(x, y) diff --git a/nlmod/dims/grid.py b/nlmod/dims/grid.py index c74294c4..2c68192b 100644 --- a/nlmod/dims/grid.py +++ b/nlmod/dims/grid.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Module containing model grid functions. - project data on different grid types @@ -28,7 +27,12 @@ from .. import cache, util from .base import _get_structured_grid_ds, _get_vertex_grid_ds, extrapolate_ds -from .layers import fill_nan_top_botm_kh_kv, get_first_active_layer, set_idomain +from .layers import ( + fill_nan_top_botm_kh_kv, + get_first_active_layer, + get_idomain, + remove_inactive_layers, +) from .rdp import rdp from .resample import ( affine_transform_gdf, @@ -40,6 +44,54 @@ logger = logging.getLogger(__name__) +def snap_extent(extent, delr, delc): + """ + snap the extent in such a way that an integer number of columns and rows fit + in the extent. The new extent is always equal to, or bigger than the + original extent. + + Parameters + ---------- + extent : list, tuple or np.array + original extent (xmin, xmax, ymin, ymax) + delr : int or float, + cell size along rows, equal to dx + delc : int or float, + cell size along columns, equal to dy + + Returns + ------- + extent : list, tuple or np.array + adjusted extent + """ + extent = list(extent).copy() + + logger.debug(f"redefining extent: {extent}") + + if delr <= 0 or delc <= 0: + raise ValueError("delr and delc should be positive values") + + # if xmin can be divided by delr do nothing, otherwise rescale xmin + if not extent[0] % delr == 0: + extent[0] -= extent[0] % delr + + # get number of columns and xmax + ncol = int(np.ceil((extent[1] - extent[0]) / delr)) + extent[1] = extent[0] + (ncol * delr) # round xmax up to close grid + + # if ymin can be divided by delc do nothing, otherwise rescale ymin + if not extent[2] % delc == 0: + extent[2] -= extent[2] % delc + + # get number of rows and ymax + nrow = int(np.ceil((extent[3] - extent[2]) / delc)) + extent[3] = extent[2] + (nrow * delc) # round ymax up to close grid + + logger.debug(f"new extent is {extent} and has {nrow} rows and {ncol} columns") + + return extent + + def xy_to_icell2d(xy, ds): """get the icell2d value of a point defined by its x and y coordinates. @@ -169,7 +221,7 @@ def modelgrid_to_vertex_ds(mg, ds, nodata=-1): for i in range(mg.ncpl): icvert[i, : cell2d[i][3]] = cell2d[i][4:] ds["icvert"] = ("icell2d", "icv"), icvert - ds["icvert"].attrs["_FillValue"] = nodata + ds["icvert"].attrs["nodata"] = nodata return ds @@ -224,6 +276,47 @@ def modelgrid_to_ds(mg): return ds +def get_dims_coords_from_modelgrid(mg): + """Get dimensions and coordinates from modelgrid. + + Used to build new xarray.DataArrays with appropriate dimensions and coordinates. + + Parameters + ---------- + mg : flopy.discretization.Grid + flopy modelgrid object + + Returns + ------- + dims : tuple of str + tuple containing dimensions + coords : dict + dictionary containing spatial coordinates derived from modelgrid + + Raises + ------ + ValueError + for unsupported grid types + """ + if mg.grid_type == "structured": + layers = np.arange(mg.nlay) + x, y = mg.xycenters # local coordinates + if mg.angrot == 0.0: + x += mg.xoffset # convert to global coordinates + y += mg.yoffset # convert to global coordinates + coords = {"layer": layers, "y": y, "x": x} + dims = ("layer", "y", "x") + elif mg.grid_type == "vertex": + layers = np.arange(mg.nlay) + y = mg.ycellcenters + x = mg.xcellcenters + coords = {"layer": layers, "y": ("icell2d", y), "x": ("icell2d", x)} + dims = ("layer", "icell2d") + else: + raise ValueError(f"grid type '{mg.grid_type}' not supported.") + return dims, coords + + def gridprops_to_vertex_ds(gridprops, ds, nodata=-1): """Gridprops is a dictionary containing keyword arguments needed to generate a flopy modelgrid instance.""" @@ -237,7 +330,7 @@ def gridprops_to_vertex_ds(gridprops, ds, nodata=-1): for i in range(gridprops["ncpl"]): icvert[i, : cell2d[i][3]] = cell2d[i][4:] ds["icvert"] = ("icell2d", "icv"), icvert - ds["icvert"].attrs["_FillValue"] = nodata + ds["icvert"].attrs["nodata"] = nodata return ds @@ -259,8 +352,8 @@ def get_cell2d_from_ds(ds): x = ds["x"].data y = ds["y"].data icvert = ds["icvert"].data - if "_FillValue" in ds["icvert"].attrs: - nodata = ds["icvert"].attrs["_FillValue"] + if "nodata" in ds["icvert"].attrs: + nodata = ds["icvert"].attrs["nodata"] else: nodata = -1 icvert = icvert.copy() @@ -303,8 +396,8 @@ def refine( If False nan layers are kept which might be usefull if you want to keep some layers that exist in other models. The default is True. model_coordinates : bool, optional - When model_coordinates is True, the features supplied in refinement features are - allready in model-coordinates. Only used when a grid is rotated. The default is + When model_coordinates is True, the features supplied in refinement_features are + already in model-coordinates. Only used when a grid is rotated. The default is False. Returns @@ -343,7 +436,9 @@ def refine( ds_has_rotation = "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0 if model_coordinates: if not ds_has_rotation: - raise (Exception("The supplied shapes need to be in realworld coordinates")) + raise ( + ValueError("The supplied shapes need to be in realworld coordinates") + ) elif ds_has_rotation: affine_matrix = get_affine_world_to_mod(ds).to_shapely() @@ -354,7 +449,9 @@ def refine( fname, geom_type, level = refinement_feature if not model_coordinates and ds_has_rotation: raise ( - Exception("Converting files to model coordinates not supported") + NotImplementedError( + "Converting files to model coordinates not supported" + ) ) g.add_refinement_features(fname, geom_type, level, layers=[0]) elif len(refinement_feature) == 2: @@ -382,8 +479,8 @@ def refine( gridprops = g.get_gridprops_disv() gridprops["area"] = g.get_area() ds = ds_to_gridprops(ds, gridprops=gridprops) - # recalculate idomain, as the interpolation changes idomain to floats - ds = set_idomain(ds, remove_nan_layers=remove_nan_layers) + if remove_nan_layers: + ds = remove_inactive_layers(ds) return ds @@ -417,6 +514,15 @@ def ds_to_gridprops(ds_in, gridprops, method="nearest", nodata=-1): xyi, _ = get_xyi_icell2d(gridprops) x = xr.DataArray(xyi[:, 0], dims=("icell2d",)) y = xr.DataArray(xyi[:, 1], dims=("icell2d",)) + + # drop non-numeric data variables + for key, dtype in ds_in.dtypes.items(): + if not np.issubdtype(dtype, np.number): + ds_in = ds_in.drop_vars(key) + logger.info( + f"cannot convert data variable {key} to refined dataset because of non-numeric dtype" + ) + if method in ["nearest", "linear"]: # resample the entire dataset in one line ds_out = ds_in.interp(x=x, y=y, method=method, kwargs={"fill_value": None}) @@ -669,9 +775,7 @@ def lrc_to_reclist( raise ValueError("col3 is set, but col1 and/or col2 are not!") if aux is not None: - if isinstance(aux, str): - aux = [aux] - elif isinstance(aux, (int, float)): + if isinstance(aux, (str, int, float)): aux = [aux] for i_aux in aux: @@ -764,9 +868,7 @@ def lcid_to_reclist( raise ValueError("col3 is set, but col1 and/or col2 are not!") if aux is not None: - if isinstance(aux, str): - aux = [aux] - elif isinstance(aux, (int, float)): + if isinstance(aux, (str, int, float)): aux = [aux] for i_aux in aux: @@ -782,6 +884,31 @@ def lcid_to_reclist( return reclist +def cols_to_reclist(ds, cellids, *args, cellid_column=0): + """Create a reclist for stress period data from a set of cellids. + + Parameters + ---------- + ds : xarray.Dataset + dataset with model data. Should have dimensions (layer, icell2d). + cellids : tuple of length 2 or 3 + tuple with indices of the cells that will be used to create the list. For a + structured grid, cellids represents (layer, row, column). For a vertex grid + cellid reprsents (layer, icell2d). + args : xarray.DatArray, str, int or float + the args parameter represents the data to be used as the columns in the reclist. + See col_to_list of the allowed values. + cellid_column : int, optional + Adds the cellid ((layer, row, col) or (layer, icell2d)) to the reclist in this + column number. Do not add cellid when cellid_column is None. The default is 0. + + """ + cols = [col_to_list(col, ds, cellids) for col in args] + if cellid_column is not None: + cols.insert(cellid_column, list(zip(*cellids))) + return list(zip(*cols)) + + def da_to_reclist( ds, mask, @@ -847,15 +974,17 @@ def da_to_reclist( Returns ------- reclist : list of tuples - every row consist of ((layer,icell2d), col1, col2, col3). + every row consists of ((layer, icell2d), col1, col2, col3). """ if "layer" in mask.dims: if only_active_cells: - cellids = np.where((mask) & (ds["idomain"] == 1)) - ignore_cells = np.sum((mask) & (ds["idomain"] != 1)) + idomain = get_idomain(ds) + cellids = np.where((mask) & (idomain == 1)) + ignore_cells = int(np.sum((mask) & (idomain != 1))) if ignore_cells > 0: logger.info( - f"ignore {ignore_cells} out of {np.sum(mask)} cells because idomain is inactive" + f"ignore {ignore_cells} out of {np.sum(mask.values)} cells " + "because idomain is inactive" ) else: cellids = np.where(mask) @@ -873,14 +1002,15 @@ def da_to_reclist( else: if first_active_layer: fal = get_first_active_layer(ds) - cellids = np.where((mask) & (fal != fal.attrs["_FillValue"])) + cellids = np.where((mask.squeeze()) & (fal != fal.attrs["nodata"])) layers = col_to_list(fal, ds, cellids) elif only_active_cells: - cellids = np.where((mask) & (ds["idomain"][layer] == 1)) - ignore_cells = np.sum((mask) & (ds["idomain"][layer] != 1)) + idomain = get_idomain(ds) + cellids = np.where((mask) & (idomain[layer] == 1)) + ignore_cells = int(np.sum((mask) & (idomain[layer] != 1))) if ignore_cells > 0: logger.info( - f"ignore {ignore_cells} out of {np.sum(mask)} cells because idomain is inactive" + f"ignore {ignore_cells} out of {np.sum(mask.values)} cells because idomain is inactive" ) layers = col_to_list(layer, ds, cellids) else: @@ -1077,7 +1207,6 @@ def gdf_to_da( da = util.get_da_from_da_ds(ds, dims=ds.top.dims, data=fill_value) for ind, row in gdf_agg.iterrows(): da.values[ind] = row[column] - da.attrs["_FillValue"] = fill_value return da @@ -1306,7 +1435,7 @@ def gdf_to_bool_da(gdf, ds): return da -def gdf_to_bool_ds(gdf, ds, da_name): +def gdf_to_bool_ds(gdf, ds, da_name, keep_coords=None): """convert a GeoDataFrame with polygon geometries into a model dataset with a data_array named 'da_name' in which each cell is 1 (True) if one or more geometries are (partly) in that cell. @@ -1319,6 +1448,9 @@ def gdf_to_bool_ds(gdf, ds, da_name): xarray with model data da_name : str The name of the variable with boolean data in the ds_out + keep_coords : tuple or None, optional + the coordinates in ds the you want keep in your empty ds. If None all + coordinates are kept from original ds. The default is None. Returns ------- @@ -1326,7 +1458,7 @@ def gdf_to_bool_ds(gdf, ds, da_name): Dataset with a single DataArray, this DataArray is 1 if polygon is in cell, 0 otherwise. Grid dimensions according to ds and mfgrid. """ - ds_out = util.get_ds_empty(ds) + ds_out = util.get_ds_empty(ds, keep_coords=keep_coords) ds_out[da_name] = gdf_to_bool_da(gdf, ds) return ds_out @@ -1366,7 +1498,7 @@ def gdf_to_grid( The GeoDataFrame with the geometries per grid-cell. """ if ml is None and ix is None: - raise (Exception("Either specify ml or ix")) + raise (ValueError("Either specify ml or ix")) if ml is not None: if isinstance(ml, xr.Dataset): @@ -1563,7 +1695,7 @@ def get_vertices(ds, vert_per_cid=4, epsilon=0, rotated=False): @cache.cache_netcdf -def mask_model_edge(ds, idomain): +def mask_model_edge(ds, idomain=None): """get data array which is 1 for every active cell (defined by idomain) at the boundaries of the model (xmin, xmax, ymin, ymax). Other cells are 0. @@ -1571,14 +1703,17 @@ def mask_model_edge(ds, idomain): ---------- ds : xarray.Dataset dataset with model data. - idomain : xarray.DataArray - idomain used to get active cells and shape of DataArray + idomain : xarray.DataArray, optional + idomain used to get active cells and shape of DataArray. Calculate from ds when + None. The default is None. Returns ------- ds_out : xarray.Dataset dataset with edge mask array """ + ds = ds.copy() # avoid side effects + # add constant head cells at model boundaries if "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0: raise NotImplementedError("model edge not yet calculated for rotated grids") @@ -1595,6 +1730,8 @@ def mask_model_edge(ds, idomain): mask2d = ymin | ymax | xmin | xmax # assign 1 to cells that are on the edge and have an active idomain + if idomain is None: + idomain = get_idomain(ds) ds_out["edge_mask"] = xr.zeros_like(idomain) for lay in ds.layer: ds_out["edge_mask"].loc[lay] = np.where( diff --git a/nlmod/dims/layers.py b/nlmod/dims/layers.py index af12af55..d540674e 100644 --- a/nlmod/dims/layers.py +++ b/nlmod/dims/layers.py @@ -5,6 +5,7 @@ import numpy as np import xarray as xr +from ..util import LayerError from . import resample logger = logging.getLogger(__name__) @@ -39,7 +40,7 @@ def calculate_thickness(ds, top="top", bot="botm"): if ds[top].shape[-1] == ds[bot].shape[-1]: # top is only top of first layer thickness = xr.zeros_like(ds[bot]) - for lay in range(len(thickness)): + for lay, _ in enumerate(thickness): if lay == 0: thickness[lay] = ds[top] - ds[bot][lay] else: @@ -47,6 +48,7 @@ def calculate_thickness(ds, top="top", bot="botm"): else: raise ValueError("2d top should have same last dimension as bot") if isinstance(ds[bot], xr.DataArray): + thickness.name = "thickness" if hasattr(ds[bot], "long_name"): thickness.attrs["long_name"] = "thickness" if hasattr(ds[bot], "standard_name"): @@ -60,6 +62,131 @@ def calculate_thickness(ds, top="top", bot="botm"): return thickness +def calculate_transmissivity( + ds, kh="kh", thickness="thickness", top="top", botm="botm" +): + """calculate the transmissivity (T) as the product of the horizontal + conductance (kh) and the thickness (D). + + Parameters + ---------- + ds : xarray.Dataset + dataset containing information about top and bottom elevations + of layers + kh : str, optional + name of data variable containing horizontal conductivity, by default 'kh' + thickness : str, optional + name of data variable containing thickness, if this data variable does not exists + thickness is calculated using top and botm. By default 'thickness' + top : str, optional + name of data variable containing tops, only used to calculate thickness if not + available in dataset. By default "top" + botm : str, optional + name of data variable containing bottoms, only used to calculate thickness if not + available in dataset. By default "botm" + + Returns + ------- + T : xarray.DataArray + DataArray containing transmissivity (T). NaN where layer thickness is zero + """ + + if thickness in ds: + thickness = ds[thickness] + else: + thickness = calculate_thickness(ds, top=top, bot=botm) + + # nan where layer does not exist (thickness is 0) + thickness_nan = xr.where(thickness == 0, np.nan, thickness) + + # calculate transmissivity + T = thickness_nan * ds[kh] + + if hasattr(T, "long_name"): + T.attrs["long_name"] = "transmissivity" + if hasattr(T, "standard_name"): + T.attrs["standard_name"] = "T" + if hasattr(thickness, "units"): + if hasattr(ds[kh], "units"): + if ds[kh].units == "m/day" and thickness.units in ["m", "mNAP"]: + T.attrs["units"] = "m2/day" + else: + T.attrs["units"] = "" + else: + T.attrs["units"] = "" + + return T + + +def calculate_resistance(ds, kv="kv", thickness="thickness", top="top", botm="botm"): + """calculate vertical resistance (c) between model layers from the vertical + conductivity (kv) and the thickness. The resistance between two layers is assigned + to the top layer. The bottom model layer gets a resistance of infinity. + + Parameters + ---------- + ds : xarray.Dataset + dataset containing information about top and bottom elevations + of layers + kv : str, optional + name of data variable containing vertical conductivity, by default 'kv' + thickness : str, optional + name of data variable containing thickness, if this data variable does not exists + thickness is calculated using top and botm. By default 'thickness' + top : str, optional + name of data variable containing tops, only used to calculate thickness if not + available in dataset. By default "top" + botm : str, optional + name of data variable containing bottoms, only used to calculate thickness if not + available in dataset. By default "botm" + + Returns + ------- + c : xarray.DataArray + DataArray containing vertical resistance (c). NaN where layer thickness is zero + """ + + if thickness in ds: + thickness = ds[thickness] + else: + thickness = calculate_thickness(ds, top=top, bot=botm) + + # nan where layer does not exist (thickness is 0) + thickness_nan = xr.where(thickness == 0, np.nan, thickness) + kv_nan = xr.where(thickness == 0, np.nan, ds[kv]) + + # backfill thickness and kv to get the right value for the layer below + thickness_bfill = thickness_nan.bfill(dim="layer") + kv_bfill = kv_nan.bfill(dim="layer") + + # calculate resistance + c = xr.zeros_like(thickness) + for ilay in range(ds.dims["layer"] - 1): + ctop = (thickness_nan.sel(layer=ds.layer[ilay]) * 0.5) / kv_nan.sel( + layer=ds.layer[ilay] + ) + cbot = (thickness_bfill.sel(layer=ds.layer[ilay + 1]) * 0.5) / kv_bfill.sel( + layer=ds.layer[ilay + 1] + ) + c[ilay] = ctop + cbot + c[ilay + 1] = np.inf + + if hasattr(c, "long_name"): + c.attrs["long_name"] = "resistance" + if hasattr(c, "standard_name"): + c.attrs["standard_name"] = "c" + if hasattr(thickness, "units"): + if hasattr(ds[kv], "units"): + if ds[kv].units == "m/day" and thickness.units in ["m", "mNAP"]: + c.attrs["units"] = "day" + else: + c.attrs["units"] = "" + else: + c.attrs["units"] = "" + + return c + + def split_layers_ds( ds, split_dict, layer="layer", top="top", bot="botm", return_reindexer=False ): @@ -116,7 +243,7 @@ def split_layers_ds( layers_org = layers.copy() # add extra layers (keep the original ones for now, as we will copy data first) for lay0 in split_dict: - for i in range(len(split_dict[lay0])): + for i, _ in enumerate(split_dict[lay0]): index = layers.index(lay0) layers.insert(index, lay0 + "_" + str(i + 1)) layers_org.insert(index, lay0) @@ -125,13 +252,15 @@ def split_layers_ds( # calclate a new top and botm, and fill other variables with original data th = calculate_thickness(ds, top=top, bot=bot) for lay0 in split_dict: + logger.info(f"Split '{lay0}' into {len(split_dict[lay0])} sub-layers") th0 = th.loc[lay0] for var in ds: if layer not in ds[var].dims: continue if lay0 == list(split_dict)[0] and var not in [top, bot]: logger.info( - f"Fill values of variable '{var}' of splitted layers with the values from the original layer." + f"Fill values for variable '{var}' in split" + " layers with the values from the original layer." ) ds = _split_var(ds, var, lay0, th0, split_dict[lay0], top, bot) @@ -554,15 +683,22 @@ def set_model_top(ds, top, min_thickness=0.0): The model dataset, containing the new top. """ if "gridtype" not in ds.attrs: - raise (Exception("Make sure the Dataset is build by nlmod")) + raise ( + KeyError( + "Dataset does not have attribute 'gridtype'. " + "Either add attribute or use nlmod functions to build the dataset." + ) + ) + if "layer" in ds["top"].dims: + raise (ValueError("set_model_top does not support top with a layer dimension")) if isinstance(top, (float, int)): top = xr.full_like(ds["top"], top) if not top.shape == ds["top"].shape: raise ( - Exception("Please make sure the new top has the same shape as the old top") + ValueError("Please make sure the new top has the same shape as the old top") ) if np.any(np.isnan(top)): - raise (Exception("Please make sure the new top does not contain nans")) + raise (ValueError("Please make sure the new top does not contain nans")) # where the botm is equal to the top, the layer is inactive # set the botm to the new top at these locations ds["botm"] = ds["botm"].where(ds["botm"] != ds["top"], top) @@ -570,14 +706,16 @@ def set_model_top(ds, top, min_thickness=0.0): ds["botm"] = ds["botm"].where(top - ds["botm"] > min_thickness, top) # change the current top ds["top"] = top - # recalculate idomain - ds = set_idomain(ds) + # remove inactive layers + ds = remove_inactive_layers(ds) return ds def set_layer_top(ds, layer, top): """Set the top of a layer.""" assert layer in ds.layer + if "layer" in ds["top"].dims: + raise (ValueError("set_layer_top does not support top with a layer dimension")) lay = np.where(ds.layer == layer)[0][0] if lay == 0: # change the top of the model @@ -595,13 +733,14 @@ def set_layer_top(ds, layer, top): ) # make sure the botms of lower layers are lower than top ds["botm"][lay:] = ds["botm"][lay:].where(ds["botm"][lay:] < top, top) - ds = set_idomain(ds) return ds def set_layer_botm(ds, layer, botm): """Set the bottom of a layer.""" assert layer in ds.layer + if "layer" in ds["top"].dims: + raise (ValueError("set_layer_botm does not support top with a layer dimension")) lay = np.where(ds.layer == layer)[0][0] # if lay > 0 and np.any(botm > ds["botm"][lay - 1]): # raise (Exception("set_layer_botm cannot change botm of higher layers yet")) @@ -611,8 +750,6 @@ def set_layer_botm(ds, layer, botm): mask = ds["botm"][lay + 1 :] < botm ds["botm"][lay + 1 :] = ds["botm"][lay + 1 :].where(mask, botm) # make sure the botm of the layers above is lever lower than the new botm - - ds = set_idomain(ds) return ds @@ -648,6 +785,39 @@ def set_minimum_layer_thickness(ds, layer, min_thickness, change="botm"): return ds +def remove_thin_layers(ds, min_thickness=0.1): + """Remove layers from cells with a thickness less than min_thickness + + The thickness of the removed cells is added to the first active layer below + """ + if "layer" in ds["top"].dims: + msg = "remove_thin_layers does not support top with a layer dimension" + raise (ValueError(msg)) + thickness = calculate_thickness(ds) + for lay_org in range(len(ds.layer)): + # determine where the layer is too thin + mask = (thickness[lay_org] > 0) & (thickness[lay_org] < min_thickness) + if mask.any(): + # we will set the botm to the top in these cells, so we first get the top + if lay_org == 0: + top = ds["top"] + else: + top = ds["botm"][lay_org - 1] + # loop over the layers, starting from lay_org + for lay in range(lay_org, len(ds.layer)): + if lay > lay_org: + # only keep cells in mask that had no thickness to begin with + # we need to increase the botm in these cells as well + mask = mask & (thickness[lay + 1] <= 0) + if not mask.any(): + break + # set the botm equal to the top in the cells in mask + ds["botm"][lay].data[mask] = top.data[mask] + # calculate the thickness again, using the new botms + thickness = calculate_thickness(ds) + return ds + + def get_kh_kv(kh, kv, anisotropy, fill_value_kh=1.0, fill_value_kv=0.1, idomain=None): """create kh en kv grid data for flopy from existing kh, kv and anistropy grids with nan values (typically from REGIS). @@ -699,7 +869,7 @@ def get_kh_kv(kh, kv, anisotropy, fill_value_kh=1.0, fill_value_kv=0.1, idomain= logger.info(f"kv and kh both undefined in layer {layer}") # fill kh by kv * anisotropy - msg_suffix = f" of kh by multipying kv by an anisotropy of {anisotropy}" + msg_suffix = f" of kh by multipying kv with an anisotropy of {anisotropy}" kh = _fill_var(kh, kv * anisotropy, idomain, msg_suffix) # fill kv by kh / anisotropy @@ -807,7 +977,7 @@ def fill_nan_top_botm_kh_kv( Steps: 1. Compute top and botm values, by filling nans by data from other layers - 2. Compute idomain from the layer thickness + 2. Remove inactive layers, with no positive thickness anywhere 3. Compute kh and kv, filling nans with anisotropy or fill_values """ @@ -815,69 +985,126 @@ def fill_nan_top_botm_kh_kv( ds = fill_top_and_bottom(ds) # 2 - ds = set_idomain(ds, remove_nan_layers=remove_nan_layers) + if remove_nan_layers: + # remove inactive layers + ds = remove_inactive_layers(ds) # 3 + idomain = get_idomain(ds) ds["kh"], ds["kv"] = get_kh_kv( ds["kh"], ds["kv"], anisotropy, fill_value_kh=fill_value_kh, fill_value_kv=fill_value_kv, - idomain=ds["idomain"], + idomain=idomain, ) return ds -def fill_top_and_bottom(ds): - """Remove Nans in botm variable, and change top from 3d to 2d if needed.""" +def fill_top_and_bottom(ds, drop_layer_dim_from_top=True): + """ + Remove Nans in botm variable, and change top from 3d to 2d if necessary. + + Parameters + ---------- + ds : xr.DataSet + model DataSet + drop_layer_dim_from_top : bool, optional + If True and top contains a layer dimension, set top to the top of the upper + layer (line the definition in MODFLOW). This removes redundant data, as the top + of all layers exept the most upper one is also defined as the bottom of previous + layers. The default is True. + + Returns + ------- + ds : xarray.Dataset + dataset with filled top and bottom data according to modflow definition, + with 2d top and 3d bottom. + """ + if "layer" in ds["top"].dims: - ds["top"] = ds["top"].max("layer") - top = ds["top"].data + top_max = ds["top"].max("layer") + if drop_layer_dim_from_top: + ds["top"] = top_max + else: + top_max = ds["top"] + botm = ds["botm"].data # remove nans from botm for lay in range(botm.shape[0]): mask = np.isnan(botm[lay]) if lay == 0: - # by setting the botm to top - botm[lay, mask] = top[mask] + # by setting the botm to top_max + botm[lay, mask] = top_max.data[mask] else: # by setting the botm to the botm of the layer above botm[lay, mask] = botm[lay - 1, mask] + if "layer" in ds["top"].dims: + # remove nans from top by setting it equal to botm + # which sets the layer thickness to 0 + top = ds["top"].data + mask = np.isnan(top) + top[mask] = botm[mask] + return ds -def set_idomain(ds, remove_nan_layers=True): - """Set idmomain in a model Dataset. +def remove_inactive_layers(ds): + """ + Remove layers which only contain inactive cells Parameters ---------- ds : xr.Dataset The model Dataset. - remove_nan_layers : bool, optional - Removes layers which only contain inactive cells. The default is True. Returns ------- ds : xr.Dataset - Dataset with added idomain-variable. + The model Dataset without inactive layers. + + """ + idomain = get_idomain(ds) + # only keep layers with at least one active cell + ds = ds.sel(layer=(idomain > 0).any(idomain.dims[1:])) + return ds + + +def get_idomain(ds): + """Get idomain from a model Dataset. + + Idomain is calculated from the thickness of the layers, and will be 1 for all layers + with a positive thickness, and -1 (pass-through) otherwise. On top of this, an + "active_domain" DataArray is applied, which is taken from ds, and can be 2d or 3d. + Idomain is set to 0 where "active_domain" is False or 0. + + + Parameters + ---------- + ds : xr.Dataset + The model Dataset. + + Returns + ------- + ds : xr.DataArray + DataArray of idomain-variable. """ # set idomain with a default of -1 (pass-through) - ds["idomain"] = xr.full_like(ds["botm"], -1, int) + idomain = xr.full_like(ds["botm"], -1, int) + idomain.name = None # drop attributes inherited from botm - ds["idomain"].attrs.clear() + idomain.attrs.clear() # set idomain of cells with a positive thickness to 1 thickness = calculate_thickness(ds) - ds["idomain"].data[thickness.data > 0.0] = 1 + idomain.data[thickness.data > 0.0] = 1 + # set idomain above/below the first/last active layer to 0 + idomain.data[idomain.where(idomain > 0).ffill(dim="layer").isnull()] = 0 + idomain.data[idomain.where(idomain > 0).bfill(dim="layer").isnull()] = 0 # set idomain to 0 in the inactive part of the model - if "active" in ds: - ds["idomain"] = ds["idomain"].where(ds["active"], 0) - if remove_nan_layers: - # only keep layers with at least one active cell - ds = ds.sel(layer=(ds["idomain"] > 0).any(ds["idomain"].dims[1:])) - # TODO: set idomain above/below the first/last active layer to 0 - # TODO: remove 'active' and replace by logic of keeping inactive cells in idomain - return ds + if "active_domain" in ds: + idomain = idomain.where(ds["active_domain"], 0) + return idomain def get_first_active_layer(ds, **kwargs): @@ -886,7 +1113,7 @@ def get_first_active_layer(ds, **kwargs): Parameters ---------- ds : xr.DataSet - Model Dataset with a variable idomain. + Model Dataset **kwargs : dict Kwargs are passed on to get_first_active_layer_from_idomain. @@ -896,11 +1123,12 @@ def get_first_active_layer(ds, **kwargs): raster in which each cell has the zero based number of the first active layer. Shape can be (y, x) or (icell2d) """ - return get_first_active_layer_from_idomain(ds["idomain"], **kwargs) + idomain = get_idomain(ds) + return get_first_active_layer_from_idomain(idomain, **kwargs) def get_first_active_layer_from_idomain(idomain, nodata=-999): - """get the first active layer in each cell from the idomain. + """get the first (top) active layer in each cell from the idomain. Parameters ---------- @@ -925,10 +1153,77 @@ def get_first_active_layer_from_idomain(idomain, nodata=-999): i, first_active_layer, ) - first_active_layer.attrs["_FillValue"] = nodata + first_active_layer.attrs["nodata"] = nodata return first_active_layer +def get_last_active_layer_from_idomain(idomain, nodata=-999): + """get the last (bottom) active layer in each cell from the idomain. + + Parameters + ---------- + idomain : xr.DataArray + idomain. Shape can be (layer, y, x) or (layer, icell2d) + nodata : int, optional + nodata value. used for cells that are inactive in all layers. + The default is -999. + + Returns + ------- + last_active_layer : xr.DataArray + raster in which each cell has the zero based number of the last + active layer. Shape can be (y, x) or (icell2d) + """ + logger.debug("get last active modellayer for each cell in idomain") + + last_active_layer = xr.where(idomain[-1] == 1, 0, nodata) + for i in range(idomain.shape[0] - 2, -1, -1): + last_active_layer = xr.where( + (last_active_layer == nodata) & (idomain[i] == 1), + i, + last_active_layer, + ) + last_active_layer.attrs["nodata"] = nodata + return last_active_layer + + +def get_layer_of_z(ds, z, above_model=-999, below_model=-999): + """Get the layer of a certain z-value in all cells from a model ds. + + Parameters + ---------- + ds : xr.DataSet + Model Dataset + z : float or xr.DataArray + The z-value for which the layer is determined + above_model : int, optional + value used for cells where z is above the top of the model. The default is -999. + below_model : int, optional + value used for cells where z is below the top of the model. The default is -999. + + Returns + ------- + layer : xr.DataArray + DataArray with values representing the integer layer index. Shape can be (y, x) + or (icell2d) + """ + layer = xr.where(ds["botm"][0] < z, 0, below_model) + for i in range(1, len(ds.layer)): + layer = xr.where((layer == below_model) & (ds["botm"][i] < z), i, layer) + + # set layer to nodata where z is above top + assert "layer" not in ds["top"].dims + layer = xr.where(ds["top"] > z, layer, above_model) + + # set nodata attribute + layer.attrs["above_model"] = above_model + layer.attrs["below_model"] = below_model + + # drop layer coordinates, as it is inherited from one of the actions above + layer = layer.drop_vars("layer") + return layer + + def update_idomain_from_thickness(idomain, thickness, mask): """get new idomain from thickness in the cells where mask is 1 (or True). @@ -956,7 +1251,7 @@ def update_idomain_from_thickness(idomain, thickness, mask): (layer, y, x) or (layer, icell2d). """ warnings.warn( - "update_idomain_from_thickness is deprecated. Please use set_idomain instead.", + "update_idomain_from_thickness is deprecated. Please use get_idomain instead.", DeprecationWarning, ) for ilay, thick in enumerate(thickness): @@ -1040,3 +1335,271 @@ def aggregate_by_weighted_mean_to_ds(ds, source_ds, var_name): ) return xr.concat(agg_ar, ds.layer) + + +def check_elevations_consistency(ds): + if "layer" in ds["top"].dims: + tops = ds["top"].data + top_ref = np.full(tops.shape[1:], np.NaN) + for lay, layer in zip(range(tops.shape[0]), ds.layer.data): + top = tops[lay] + mask = ~np.isnan(top) + higher = top[mask] > top_ref[mask] + if np.any(higher): + n = int(higher.sum()) + logger.warning( + f"The top of layer {layer} is higher than the top of a previous layer in {n} cells" + ) + top_ref[mask] = top[mask] + + bots = ds["botm"].data + bot_ref = np.full(bots.shape[1:], np.NaN) + for lay, layer in zip(range(bots.shape[0]), ds.layer.data): + bot = bots[lay] + mask = ~np.isnan(bot) + higher = bot[mask] > bot_ref[mask] + if np.any(higher): + n = int(higher.sum()) + logger.warning( + f"The bottom of layer {layer} is higher the bottom of a previous layer in {n} cells" + ) + bot_ref[mask] = bot[mask] + + thickness = calculate_thickness(ds) + mask = thickness < 0.0 + if mask.any(): + logger.warning(f"Thickness of layers is negative in {mask.sum()} cells.") + + +def insert_layer(ds, name, top, bot, kh=None, kv=None, copy=True): + """ + Inserts a layer in a model Dataset, burning it in an existing layer model. + + This method loops over the existing layers, and checks if (part of) the new layer + needs to be inserted above the existing layer, and if the top or bottom of the + existing layer needs to be altered. + + For now, this method needs a layer model with a 3d-top, like you get using the + method `nlmod.read.get_regis()`, and does not function for a model Dataset with a 2d + (structured) or 1d (vertex) top. + + When comparing the height of the new layer with an existing layer, there are 7 + options: + + 1 The new layer is entirely above the existing layer: layer is added completely + above existing layer. When the bottom of the new layer is above the top of the + existing layer (which can happen for the first layer), this creates a gap in the + layer model. + + 2 part of the new layer lies within an existing layer, bottom is never below: layer + is added above the existing layer, and the top of existing layer is lowered. + + 3 there are locations where the new layer is above the bottom of the existing layer, + but below the top of the existing layer. The new layer splits the existing layer + into two sub-layers. This is not supported (yet) and raises an Exception. + + 4 part of the new layer lies above the bottom of the existing layer, while at other + locations the new layer is below the existing layer. The new layer is split, part + of the layer is added above the existing layer, and part of the new layer is added + to the layer model in the next iteration(s) (above the next layer). + + 5 Only the upper part of the new layer overlaps with the existing layer: the layer + is not added above the extsing layer, but the bottom of the existing layer is + raised because of the overlap. + + 6 The new layer is below the existing layer everywhere. Nothing happens, move on to + the next existing layer. + + 7 When (part of) the new layer is not added to the layer model after comparison + with the last existing layer, the (remaining part of) the new layer is added below + the existing layers, at the bottom of the model. + + Parameters + ---------- + ds : xarray.Dataset + xarray Dataset containing information about layers + name : string + The name of the new layer. + top : xr.DataArray + The top of the new layer. + bot : xr.DataArray + The bottom of the new layer.. + kh : xr.DataArray, optional + The horizontal conductivity of the new layer. The default is None. + kv : xr.DataArray, optional + The vertical conductivity of the new layer. The default is None. + copy : bool, optional + If copy=True, data in the return value is always copied, so the original Dataset + is not altered. The default is True. + + Returns + ------- + ds : xarray.Dataset + xarray Dataset containing the new layer(s) + + """ + shape = ds["botm"].shape[1:] + assert top.shape == shape + assert bot.shape == shape + msg = "Inserting layers is only supported with a 3d top for now" + assert "layer" in ds["top"].dims, msg + if kh is not None: + assert kh.shape == shape + if kv is not None: + assert kv.shape == shape + todo = ~(np.isnan(top.data) | np.isnan(bot.data)) & ((top - bot).data > 0) + if not todo.any(): + logger.warning(f"Thickness of new layer {name} is never larger than 0") + if copy: + # make a copy, so we are sure we do not alter the original DataSet + ds = ds.copy(deep=True) + isplit = None + for layer in ds.layer.data: + if not todo.any(): + continue + # determine the top and bottom of layer, taking account they could be NaN + # we assume a zero thickness when top or bottom is NaN + top_layer = ds["top"].loc[layer:].max("layer").data + bot_layer = ds["botm"].loc[layer].data + mask = np.isnan(bot_layer) + bot_layer[mask] = top_layer[mask] + + top_higher_than_bot = top.data > bot_layer + if not top_higher_than_bot[todo].any(): + # 6 the new layer is entire below the existing layer, do nothing + continue + bot_lower_than_top = bot.data < top_layer + bot_lower_than_bot = bot.data < bot_layer + if not bot_lower_than_top[todo].any(): + # 1 the new layer can be added on top of the existing layer + if isplit is not None: + isplit += 1 + ds = _insert_layer_above( + ds, layer, name, isplit, todo, top, bot, kh, kv, copy + ) + todo[todo] = False + continue + # do not increase top of layer to bottom of new layer + if bot_lower_than_top[todo].any(): + # the new layer can be added on top of the existing layer, + # possibly only partly + if not bot_lower_than_bot[todo].any(): + # 2 the top of the existing layer needs to be lowered + mask = todo & bot_lower_than_top + new_top_layer = ds["top"].loc[layer] + new_top_layer.data[mask] = bot.data[mask] + ds["top"].loc[layer] = new_top_layer + # the new layer can be added on top of the existing layer + if isplit is not None: + isplit += 1 + ds = _insert_layer_above( + ds, layer, name, isplit, todo, top, bot, kh, kv, copy + ) + todo[todo] = False + continue + if not bot_lower_than_bot[todo].all(): + bot_higher_than_bot = bot.data > bot_layer + if not bot_higher_than_bot[todo].any(): + continue + top_lower_than_top = top.data < top_layer + if (todo & bot_higher_than_bot & top_lower_than_top).any(): + # 3 the existing layer needs to be split, + # as part of it is below and part is above the new layer + msg = ( + f"Existing layer {layer} exists in some cells both above and " + f"below the inserted layer {name}. Therefore existing layer " + f"{layer} needs to be split in two, which is not supported." + ) + raise (LayerError(msg)) + # 4 the new layer needs to be split, as part of the new layer is + # above the bottom of the existing layer, and part of it is below the + # existing layer + if isplit is None: + isplit = 1 + else: + isplit += 1 + # the top of the existing layer needs to be lowered + mask = todo & bot_higher_than_bot & bot_lower_than_top + new_top_layer = ds["top"].loc[layer] + new_top_layer.data[mask] = bot.data[mask] + ds["top"].loc[layer] = new_top_layer + # and we insert the new layer + mask = todo & bot_higher_than_bot + ds = _insert_layer_above( + ds, layer, name, isplit, mask, top, bot, kh, kv, copy + ) + todo[mask] = False + + mask = todo & top_higher_than_bot + if mask.any(): + # 5 when the new layer is not added above the existing layer, as the bottom + # of the new layer is always lower than the bottom of the existing + # layer: the bottom of the existing layer needs to be raised to the top + # of the new layer + new_bot_layer = ds["botm"].loc[layer] + new_bot_layer.data[mask] = top.data[mask] + ds["botm"].loc[layer] = new_bot_layer + + if todo.any(): + # 7 the new layer needs to be added to the bottom of the model + if isplit is not None: + isplit += 1 + ds = _insert_layer_below(ds, None, name, isplit, mask, top, bot, kh, kv, copy) + return ds + + +def _insert_layer_above(ds, above_layer, name, isplit, mask, top, bot, kh, kv, copy): + new_layer_name = _get_new_layer_name(name, isplit) + layers = list(ds.layer.data) + if above_layer is None: + above_layer = layers[0] + layers.insert(layers.index(above_layer), new_layer_name) + ds = ds.reindex({"layer": layers}, copy=copy) + ds = _set_new_layer_values(ds, new_layer_name, mask, top, bot, kh, kv) + return ds + + +def _insert_layer_below(ds, below_layer, name, isplit, mask, top, bot, kh, kv, copy): + new_layer_name = _get_new_layer_name(name, isplit) + layers = list(ds.layer.data) + if below_layer is None: + below_layer = layers[-1] + layers.insert(layers.index(below_layer) + 1, new_layer_name) + ds = ds.reindex({"layer": layers}, copy=copy) + ds = _set_new_layer_values(ds, new_layer_name, mask, top, bot, kh, kv) + return ds + + +def _set_new_layer_values(ds, new_layer_name, mask, top, bot, kh, kv): + ds["top"].loc[new_layer_name].data[mask] = top.data[mask] + ds["botm"].loc[new_layer_name].data[mask] = bot.data[mask] + if kh is not None: + ds["kh"].loc[new_layer_name].data[mask] = kh.data[mask] + if kv is not None: + ds["kv"].loc[new_layer_name].data[mask] = kv.data[mask] + return ds + + +def _get_new_layer_name(name, isplit): + new_layer_name = name + if isplit is not None: + new_layer_name = new_layer_name + "_" + str(isplit) + return new_layer_name + + +def remove_layer(ds, layer): + """Removes a layer from a Dataset, without changing elevations of other layers. + + This will create gaps in the layer model. + """ + layers = list(ds.layer.data) + if layer not in layers: + raise (KeyError(f"layer '{layer}' not present in Dataset")) + if "layer" not in ds["top"].dims: + index = layers.index(layer) + if index == 0: + # lower the top to the second layer + ds["top"] = ds["botm"].loc[layers[1]] + layers.remove(layer) + ds = ds.reindex({"layer": layers}) + return ds diff --git a/nlmod/dims/resample.py b/nlmod/dims/resample.py index 7e339a8f..50a17cfe 100644 --- a/nlmod/dims/resample.py +++ b/nlmod/dims/resample.py @@ -1,8 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Fri Apr 2 15:08:50 2021. - -@author: oebbe -""" import logging import numbers @@ -208,13 +203,13 @@ def _set_angrot_attributes(extent, xorigin, yorigin, angrot, attrs): extent[0] = 0.0 extent[1] = extent[1] - xorigin elif extent[0] != 0.0: - raise (Exception("Either extent[0] or xorigin needs to be 0.0")) + raise (ValueError("Either extent[0] or xorigin needs to be 0.0")) if yorigin == 0.0: yorigin = extent[2] extent[2] = 0.0 extent[3] = extent[3] - yorigin elif extent[2] != 0.0: - raise (Exception("Either extent[2] or yorigin needs to be 0.0")) + raise (ValueError("Either extent[2] or yorigin needs to be 0.0")) attrs["extent"] = extent attrs["xorigin"] = xorigin attrs["yorigin"] = yorigin @@ -441,7 +436,7 @@ def dim_to_regular_dim(da, dims, z): # just use griddata z = griddata(points, da.data, xi, method=method) dims = ["y", "x"] - coords = dict(x=ds.x, y=ds.y) + coords = {"x": ds.x, "y": ds.y} return xr.DataArray(z, dims=dims, coords=coords) @@ -451,7 +446,7 @@ def structured_da_to_ds(da, ds, method="average", nodata=np.NaN): Parameters ---------- da : xarray.DataArray - THe data-array to be resampled, with dimensions x and y. + The data-array to be resampled, with dimensions x and y. ds : xarray.Dataset The model dataset. method : string or rasterio.enums.Resampling, optional @@ -462,7 +457,7 @@ def structured_da_to_ds(da, ds, method="average", nodata=np.NaN): method is 'linear' or 'nearest' da.interp() is used. Otherwise da.rio.reproject_match() is used. The default is "average". nodata : float, optional - THe nodata value in input and output. THe default is np.NaN. + The nodata value in input and output. The default is np.NaN. Returns ------- @@ -486,7 +481,7 @@ def structured_da_to_ds(da, ds, method="average", nodata=np.NaN): if hasattr(rasterio.enums.Resampling, method): resampling = getattr(rasterio.enums.Resampling, method) else: - raise (Exception(f"Unknown resample method: {method}")) + raise ValueError(f"Unknown resample method: {method}") # fill crs if it is None for da or ds if ds.rio.crs is None and da.rio.crs is None: logger.info("No crs in da and ds. Assuming ds and da are both in EPSG:28992") @@ -533,11 +528,11 @@ def structured_da_to_ds(da, ds, method="average", nodata=np.NaN): da_temp = da_temp.assign_coords(x=x, y=y) mask = ds["area"] == area - da_out.loc[dict(icell2d=mask)] = da_temp.sel( + da_out.loc[{"icell2d": mask}] = da_temp.sel( y=ds["y"][mask], x=ds["x"][mask] ) else: - raise (Exception(f"Gridtype {ds.gridtype} not supported")) + raise (NotImplementedError(f"Gridtype {ds.gridtype} not supported")) # some stuff is added by the reproject_match function that should not be there added_coords = set(da_out.coords) - set(ds.coords) diff --git a/nlmod/dims/time.py b/nlmod/dims/time.py index 89ffa95a..f766354a 100644 --- a/nlmod/dims/time.py +++ b/nlmod/dims/time.py @@ -1,20 +1,18 @@ -# -*- coding: utf-8 -*- -"""Created on Fri Apr 17 13:50:48 2020. - -@author: oebbe -""" - import datetime as dt import logging +import warnings import numpy as np import pandas as pd +import xarray as xr from xarray import IndexVariable +from .attributes_encodings import dim_attrs + logger = logging.getLogger(__name__) -def set_ds_time( +def set_ds_time_deprecated( ds, time=None, steady_state=False, @@ -77,6 +75,13 @@ def set_ds_time( ds : xarray.Dataset dataset with time variant model data """ + + warnings.warn( + "this function is deprecated and will eventually be removed, " + "please use nlmod.time.set_ds_time() in the future.", + DeprecationWarning, + ) + # checks if time_units.lower() != "days": raise NotImplementedError() @@ -130,9 +135,193 @@ def set_ds_time( ds.time.attrs["steady_start"] = int(steady_start) ds.time.attrs["steady_state"] = int(steady_state) + # add to ds (for new version nlmod) + # add steady, nstp and tsmult to dataset + steady = int(steady_state) * np.ones(len(time_dt), dtype=int) + if steady_start: + steady[0] = 1 + ds["steady"] = ("time",), steady + + if isinstance(nstp, (int, np.integer)): + nstp = nstp * np.ones(len(time), dtype=int) + ds["nstp"] = ("time",), nstp + + if isinstance(tsmult, float): + tsmult = tsmult * np.ones(len(time)) + ds["tsmult"] = ("time",), tsmult + + return ds + + +def set_ds_time( + ds, + start, + time=None, + steady=False, + steady_start=True, + time_units="DAYS", + perlen=None, + nstp=1, + tsmult=1.0, +): + """Set time discretisation for model dataset. + + Parameters + ---------- + ds : xarray.Dataset + model dataset + start : int, float, str or pandas.Timestamp + model start. When start is an integer or float it is interpreted as the number + of days of the first stress-period. When start is a string or pandas Timestamp + it is the start datetime of the simulation. + time : float, int or array-like, optional + float(s) (indicating elapsed time) or timestamp(s) corresponding to the end of + each stress period in the model. When time is a single value, the model will + have only one stress period. When time is None, the stress period lengths have + to be supplied via perlen. The default is None. + steady : arraylike or bool, optional + arraylike indicating which stress periods are steady-state, by default False, + which sets all stress periods to transient with the first period determined by + value of `steady_start`. + steady_start : bool, optional + whether to set the first period to steady-state, default is True, only used + when steady is passed as single boolean. + time_units : str, optional + time units, by default "DAYS" + perlen : float, int or array-like, optional + length of each stress-period. Only used when time is None. When perlen is a + single value, the model will have only one stress period. The default is None. + nstp : int or array-like, optional + number of steps per stress period, stored in ds.attrs, default is 1 + tsmult : float, optional + timestep multiplier within stress periods, stored in ds.attrs, default is 1.0 + + Returns + ------- + ds : xarray.Dataset + model dataset with added time coordinate + + """ + logger.info( + "Function set_ds_time() has changed since nlmod version 0.7." + " For the old behavior, use `nlmod.time.set_ds_time_deprecated()`." + ) + + if time is None and perlen is None: + raise (ValueError("Please specify either time or perlen in set_ds_time")) + elif perlen is not None: + if time is not None: + msg = f"Cannot use both time and perlen. Ignoring perlen: {perlen}" + logger.warning(msg) + else: + if isinstance(perlen, (int, np.integer, float)): + perlen = [perlen] + time = np.cumsum(perlen) + + if isinstance(time, str) or not hasattr(time, "__iter__"): + time = [time] + + # parse start + if isinstance(start, (int, np.integer, float)): + if isinstance(time[0], (int, np.integer, float)): + raise (ValueError("Make sure start or time contains a valid TimeStamp")) + start = time[0] - pd.to_timedelta(start, "D") + elif isinstance(start, str): + start = pd.Timestamp(start) + elif isinstance(start, (pd.Timestamp, np.datetime64)): + pass + else: + raise TypeError("Cannot parse start datetime.") + + # convert time to Timestamps + if isinstance(time[0], (int, np.integer, float)): + time = pd.Timestamp(start) + pd.to_timedelta(time, time_units) + elif isinstance(time[0], str): + time = pd.to_datetime(time) + elif isinstance(time[0], (pd.Timestamp, np.datetime64, xr.core.variable.Variable)): + pass + else: + raise TypeError("Cannot process 'time' argument. Datatype not understood.") + + if time[0] <= start: + msg = ( + "The timestamp of the first stress period cannot be before or " + "equal to the model start time! Please modify `time` or `start`!" + ) + logger.error(msg) + raise ValueError(msg) + + ds = ds.assign_coords(coords={"time": time}) + ds.coords["time"].attrs = dim_attrs["time"] + + # add steady, nstp and tsmult to dataset + if isinstance(steady, bool): + steady = int(steady) * np.ones(len(time), dtype=int) + if steady_start: + steady[0] = 1 + ds["steady"] = ("time",), steady + + if isinstance(nstp, (int, np.integer)): + nstp = nstp * np.ones(len(time), dtype=int) + ds["nstp"] = ("time",), nstp + + if isinstance(tsmult, float): + tsmult = tsmult * np.ones(len(time)) + ds["tsmult"] = ("time",), tsmult + + if time_units == "D": + time_units = "DAYS" + ds.time.attrs["time_units"] = time_units + ds.time.attrs["start"] = str(start) + return ds +def ds_time_idx_from_tdis_settings(start, perlen, nstp=1, tsmult=1.0, time_units="D"): + """Get time index from TDIS perioddata: perlen, nstp, tsmult. + + + Parameters + ---------- + start : str, pd.Timestamp + start datetime + perlen : array-like + array of period lengths + nstp : int, or array-like optional + number of steps per period, by default 1 + tsmult : float or array-like, optional + timestep multiplier per period, by default 1.0 + time_units : str, optional + time units, by default "D" + + Returns + ------- + IndexVariable + time coordinate for xarray data-array or dataset + """ + deltlist = [] + for kper, delt in enumerate(perlen): + if not isinstance(nstp, int): + kstpkper = nstp[kper] + else: + kstpkper = nstp + + if not isinstance(tsmult, float): + tsm = tsmult[kper] + else: + tsm = tsmult + + if tsm > 1.0: + delt0 = delt * (tsm - 1) / (tsm**kstpkper - 1) + delt = delt0 * tsm ** np.arange(kstpkper) + else: + delt = np.ones(kstpkper) * delt / kstpkper + deltlist.append(delt) + + dt_arr = np.cumsum(np.concatenate(deltlist)) + return ds_time_idx(dt_arr, start_datetime=start, time_units=time_units) + + def estimate_nstp( forcing, perlen=1, tsmult=1.1, nstp_min=1, nstp_max=25, return_dt_arr=False ): @@ -220,6 +409,15 @@ def estimate_nstp( def ds_time_from_model(gwf): + warnings.warn( + "this function was renamed to `ds_time_idx_from_model`. " + "Please use the new function name.", + DeprecationWarning, + ) + return ds_time_idx_from_model(gwf) + + +def ds_time_idx_from_model(gwf): """Get time index variable from model (gwf or gwt). Parameters @@ -233,10 +431,19 @@ def ds_time_from_model(gwf): time coordinate for xarray data-array or dataset """ - return ds_time_from_modeltime(gwf.modeltime) + return ds_time_idx_from_modeltime(gwf.modeltime) def ds_time_from_modeltime(modeltime): + warnings.warn( + "this function was renamed to `ds_time_idx_from_model`. " + "Please use the new function name.", + DeprecationWarning, + ) + return ds_time_idx_from_modeltime(modeltime) + + +def ds_time_idx_from_modeltime(modeltime): """Get time index variable from modeltime object. Parameters @@ -287,3 +494,39 @@ def ds_time_idx(t, start_datetime=None, time_units="D"): time.attrs["start"] = str(start_datetime) return time + + +def dataframe_to_flopy_timeseries( + df, + ds=None, + package=None, + filename=None, + time_series_namerecord=None, + interpolation_methodrecord="stepwise", + append=False, +): + assert not df.isna().any(axis=None) + if ds is not None: + # set index to days after the start of the simulation + df = df.copy() + df.index = (df.index - pd.to_datetime(ds.time.start)) / pd.Timedelta(1, "D") + # generate a list of tuples with time as the first record, followed by the columns + timeseries = [(i,) + tuple(v) for i, v in zip(df.index, df.values)] + if package is None: + return timeseries + if filename is None: + filename = f"{package.filename}_ts" + if time_series_namerecord is None: + time_series_namerecord = list(df.columns) + + if isinstance(interpolation_methodrecord, str): + interpolation_methodrecord = [interpolation_methodrecord] * len(df.columns) + + # initialize or append a new package + method = package.ts.append_package if append else package.ts.initialize + return method( + filename=filename, + timeseries=timeseries, + time_series_namerecord=time_series_namerecord, + interpolation_methodrecord=interpolation_methodrecord, + ) diff --git a/nlmod/epsg28992.py b/nlmod/epsg28992.py new file mode 100644 index 00000000..6429fd9b --- /dev/null +++ b/nlmod/epsg28992.py @@ -0,0 +1,13 @@ +""" +NOTE: this is the correct epsg:28992 definition for plotting backgroundmaps in RD +More info (in Dutch) here: +https://qgis.nl/2011/12/05/epsg28992-of-rijksdriehoekstelsel-verschuiving/ +This was still a problem in October 2023 +""" + +EPSG_28992 = ( + "+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 " + "+x_0=155000 +y_0=463000 +ellps=bessel " + "+towgs84=565.417,50.3319,465.552,-0.398957,0.343988,-1.8774,4.0725 +units=m " + "+no_defs" +) diff --git a/nlmod/gis.py b/nlmod/gis.py index b58b5628..90ecfb28 100644 --- a/nlmod/gis.py +++ b/nlmod/gis.py @@ -11,8 +11,7 @@ def vertex_da_to_gdf(model_ds, data_variables, polygons=None, dealing_with_time="mean"): - """Convert one or more DataArrays from a vertex model dataset to a - Geodataframe. + """Convert one or more DataArrays from a vertex model dataset to a Geodataframe. Parameters ---------- @@ -88,8 +87,7 @@ def vertex_da_to_gdf(model_ds, data_variables, polygons=None, dealing_with_time= def struc_da_to_gdf(model_ds, data_variables, polygons=None, dealing_with_time="mean"): - """Convert one or more DataArrays from a structured model dataset to a - Geodataframe. + """Convert one or more DataArrays from a structured model dataset to a Geodataframe. Parameters ---------- @@ -225,7 +223,6 @@ def ds_to_vector_file( # get default combination dictionary if combine_dic is None: combine_dic = { - "idomain": {"idomain"}, "topbot": {"top", "botm"}, "sea": {"northsea", "bathymetry"}, } @@ -312,8 +309,8 @@ def ds_to_ugrid_nc_file( yv="yv", face_node_connectivity="icvert", ): - """Save a model dataset to a UGRID NetCDF file, so it can be opened as a - Mesh Layer in qgis. + """Save a model dataset to a UGRID NetCDF file, so it can be opened as a Mesh Layer + in qgis. Parameters ---------- @@ -368,7 +365,7 @@ def ds_to_ugrid_nc_file( # direction. Flopy specifies them in clockwise direction, so we need to # reverse the direction. data = np.flip(ds[face_node_connectivity].data, 1) - nodata = ds[face_node_connectivity].attrs.get("_FillValue") + nodata = ds[face_node_connectivity].attrs.get("nodata") if nodata is not None: # move the nodata values from the first columns to the last data_new = np.full(data.shape, nodata) diff --git a/nlmod/gwf/gwf.py b/nlmod/gwf/gwf.py index 80cce721..abc3d30e 100644 --- a/nlmod/gwf/gwf.py +++ b/nlmod/gwf/gwf.py @@ -12,6 +12,7 @@ import xarray as xr from ..dims import grid +from ..dims.layers import get_idomain from ..sim import ims, sim, tdis from ..util import _get_value_from_ds_attr, _get_value_from_ds_datavar from . import recharge @@ -38,7 +39,7 @@ def gwf(ds, sim, under_relaxation=False, **kwargs): """ # start creating model - logger.info("creating modflow GWF") + logger.info("creating mf6 GWF") # Create the Flopy groundwater flow (gwf) model object model_nam_file = f"{ds.model_name}.nam" @@ -62,7 +63,7 @@ def gwf(ds, sim, under_relaxation=False, **kwargs): def dis(ds, gwf, length_units="METERS", pname="dis", **kwargs): - """get discretisation package from the model dataset. + """create discretisation package from the model dataset. Parameters ---------- @@ -84,7 +85,7 @@ def dis(ds, gwf, length_units="METERS", pname="dis", **kwargs): def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): - """get discretisation package from the model dataset. + """create discretisation package from the model dataset. Parameters ---------- @@ -102,7 +103,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): dis : flopy ModflowGwfdis or flopy ModflowGwtdis discretisation package. """ - logger.info("creating modflow DIS") + logger.info("creating mf6 DIS") if ds.gridtype == "vertex": return disv(ds, model, length_units=length_units) @@ -122,6 +123,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): yorigin = ds.extent[2] angrot = 0.0 + idomain = get_idomain(ds).data if model.model_type == "gwf6": dis = flopy.mf6.ModflowGwfdis( model, @@ -137,7 +139,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): delc=ds["delc"].values if "delc" in ds else ds.delc, top=ds["top"].data, botm=ds["botm"].data, - idomain=ds["idomain"].data, + idomain=idomain, filename=f"{ds.model_name}.dis", **kwargs, ) @@ -156,7 +158,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): delc=ds["delc"].values if "delc" in ds else ds.delc, top=ds["top"].data, botm=ds["botm"].data, - idomain=ds["idomain"].data, + idomain=idomain, filename=f"{ds.model_name}_gwt.dis", **kwargs, ) @@ -167,7 +169,7 @@ def _dis(ds, model, length_units="METERS", pname="dis", **kwargs): def disv(ds, gwf, length_units="METERS", pname="disv", **kwargs): - """get discretisation vertices package from the model dataset. + """create discretisation vertices package from the model dataset. Parameters ---------- @@ -189,7 +191,7 @@ def disv(ds, gwf, length_units="METERS", pname="disv", **kwargs): def _disv(ds, model, length_units="METERS", pname="disv", **kwargs): - """get discretisation vertices package from the model dataset. + """create discretisation vertices package from the model dataset. Parameters ---------- @@ -207,16 +209,12 @@ def _disv(ds, model, length_units="METERS", pname="disv", **kwargs): disv : flopy ModflowGwfdisv or flopy ModflowGwtdisv disv package """ - logger.info("creating modflow DISV") + logger.info("creating mf6 DISV") if "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0: xorigin = ds.attrs["xorigin"] yorigin = ds.attrs["yorigin"] angrot = ds.attrs["angrot"] - elif "extent" in ds.attrs.keys(): - xorigin = ds.attrs["extent"][0] - yorigin = ds.attrs["extent"][2] - angrot = 0.0 else: xorigin = 0.0 yorigin = 0.0 @@ -224,10 +222,11 @@ def _disv(ds, model, length_units="METERS", pname="disv", **kwargs): vertices = grid.get_vertices_from_ds(ds) cell2d = grid.get_cell2d_from_ds(ds) + idomain = get_idomain(ds).data if model.model_type == "gwf6": disv = flopy.mf6.ModflowGwfdisv( model, - idomain=ds["idomain"].data, + idomain=idomain, xorigin=xorigin, yorigin=yorigin, length_units=length_units, @@ -245,7 +244,7 @@ def _disv(ds, model, length_units="METERS", pname="disv", **kwargs): elif model.model_type == "gwt6": disv = flopy.mf6.ModflowGwtdisv( model, - idomain=ds["idomain"].data, + idomain=idomain, xorigin=xorigin, yorigin=yorigin, length_units=length_units, @@ -273,7 +272,7 @@ def _disv(ds, model, length_units="METERS", pname="disv", **kwargs): def npf( ds, gwf, k="kh", k33="kv", icelltype=0, save_flows=False, pname="npf", **kwargs ): - """get node property flow package from model dataset. + """create node property flow package from model dataset. Parameters ---------- @@ -306,7 +305,7 @@ def npf( npf : flopy ModflowGwfnpf npf package. """ - logger.info("creating modflow NPF") + logger.info("creating mf6 NPF") if isinstance(icelltype, str): icelltype = ds[icelltype] @@ -318,8 +317,8 @@ def npf( gwf, pname=pname, icelltype=icelltype, - k=k.data, - k33=k33.data, + k=k, + k33=k33, save_flows=save_flows, **kwargs, ) @@ -332,12 +331,12 @@ def ghb( gwf, bhead=None, cond=None, - da_name=None, pname="ghb", auxiliary=None, + layer=None, **kwargs, ): - """get general head boundary from model dataset. + """create general head boundary from model dataset. Parameters ---------- @@ -353,12 +352,13 @@ def ghb( ghb conductance, either as string pointing to data array in ds or as data array. By default None, which assumes data array is stored under "ghb_cond". - da_name : str - name of the ghb files in the model dataset. pname : str, optional package name auxiliary : str or list of str name(s) of data arrays to include as auxiliary data to reclist + layer : int or None + The layer in which the boundary is added. It is added to the first active layer + when layer is None. The default is None. Raises ------ @@ -370,36 +370,27 @@ def ghb( ghb : flopy ModflowGwfghb ghb package """ - logger.info("creating modflow GHB") + logger.info("creating mf6 GHB") - if da_name is not None: - warnings.warn( - "the kwarg 'da_name' is no longer supported, " - "specify 'bhead' and 'cond' explicitly!", - DeprecationWarning, - ) - bhead = f"{da_name}_peil" - cond = f"{da_name}_cond" - - mask_arr = _get_value_from_ds_datavar(ds, "cond", cond) - mask = mask_arr != 0 + mask_arr = _get_value_from_ds_datavar(ds, "ghb_cond", cond, return_da=True) + mask = mask_arr > 0 + first_active_layer = layer is None ghb_rec = grid.da_to_reclist( ds, mask, col1=bhead, col2=cond, - first_active_layer=True, - only_active_cells=False, - layer=0, + layer=layer, aux=auxiliary, + first_active_layer=first_active_layer, + only_active_cells=False, ) if len(ghb_rec) > 0: ghb = flopy.mf6.ModflowGwfghb( gwf, auxiliary="CONCENTRATION" if auxiliary is not None else None, - print_input=True, maxbound=len(ghb_rec), stress_period_data=ghb_rec, save_flows=True, @@ -408,7 +399,7 @@ def ghb( ) if (auxiliary is not None) and (ds.transport == 1): logger.info("-> adding GHB to SSM sources list") - ssm_sources = ds.attrs["ssm_sources"] + ssm_sources = list(ds.attrs["ssm_sources"]) if ghb.package_name not in ssm_sources: ssm_sources += [ghb.package_name] ds.attrs["ssm_sources"] = ssm_sources @@ -424,12 +415,11 @@ def drn( gwf, elev="drn_elev", cond="drn_cond", - da_name=None, pname="drn", layer=None, **kwargs, ): - """get drain from model dataset. + """create drain from model dataset. Parameters ---------- @@ -449,42 +439,34 @@ def drn( this is deprecated, name of the drn files in the model dataset pname : str, optional package name + layer : int or None + The layer in which the boundary is added. It is added to the first active layer + when layer is None. The default is None. Returns ------- drn : flopy ModflowGwfdrn drn package """ - logger.info("creating modflow DRN") - - if da_name is not None: - warnings.warn( - "the kwarg 'da_name' is no longer supported, " - "specify 'elev' and 'cond' explicitly!", - DeprecationWarning, - ) - elev = f"{da_name}_peil" - cond = f"{da_name}_cond" + logger.info("creating mf6 DRN") - mask_arr = _get_value_from_ds_datavar(ds, "cond", cond) - mask = mask_arr != 0 + mask_arr = _get_value_from_ds_datavar(ds, "cond", cond, return_da=True) + mask = mask_arr > 0 first_active_layer = layer is None - drn_rec = grid.da_to_reclist( ds, mask=mask, col1=elev, col2=cond, + layer=layer, first_active_layer=first_active_layer, only_active_cells=False, - layer=layer, ) if len(drn_rec) > 0: drn = flopy.mf6.ModflowGwfdrn( gwf, - print_input=True, maxbound=len(drn_rec), stress_period_data=drn_rec, save_flows=True, @@ -499,8 +481,176 @@ def drn( return None +def riv( + ds, + gwf, + stage="riv_stage", + cond="riv_cond", + rbot="riv_rbot", + pname="riv", + auxiliary=None, + layer=None, + **kwargs, +): + """create river package from model dataset. + + Parameters + ---------- + ds : xarray.Dataset + dataset with model data. + gwf : flopy ModflowGwf + groundwaterflow object. + stage : str or xarray.DataArray, optional + river stage, either as string pointing to data + array in ds or as data array. By default assumes + data array is stored under "riv_stage". + cond : str or xarray.DataArray, optional + river conductance, either as string pointing to data + array in ds or as data array. By default assumes + data array is stored under "riv_cond". + rbot : str or xarray.DataArray, optional + river bottom elevation, either as string pointing to data + array in ds or as data array. By default assumes + data array is stored under "riv_rbot". + pname : str, optional + package name + auxiliary : str or list of str + name(s) of data arrays to include as auxiliary data to reclist + layer : int or None + The layer in which the boundary is added. It is added to the first active layer + when layer is None. The default is None. + + Returns + ------- + riv : flopy ModflowGwfriv + riv package + """ + logger.info("creating mf6 RIV") + + mask_arr = _get_value_from_ds_datavar(ds, "cond", cond, return_da=True) + mask = mask_arr > 0 + + first_active_layer = layer is None + riv_rec = grid.da_to_reclist( + ds, + mask=mask, + col1=stage, + col2=cond, + col3=rbot, + layer=layer, + aux=auxiliary, + first_active_layer=first_active_layer, + only_active_cells=False, + ) + + if len(riv_rec) > 0: + riv = flopy.mf6.ModflowGwfriv( + gwf, + maxbound=len(riv_rec), + stress_period_data=riv_rec, + auxiliary="CONCENTRATION" if auxiliary is not None else None, + save_flows=True, + pname=pname, + **kwargs, + ) + if (auxiliary is not None) and (ds.transport == 1): + logger.info("-> adding RIV to SSM sources list") + ssm_sources = list(ds.attrs["ssm_sources"]) + if riv.package_name not in ssm_sources: + ssm_sources += [riv.package_name] + ds.attrs["ssm_sources"] = ssm_sources + return riv + else: + logger.warning("no riv pkg added") + return None + + +def chd( + ds, + gwf, + mask="chd_mask", + head="chd_head", + pname="chd", + auxiliary=None, + layer=0, + **kwargs, +): + """create constant head package from model dataset. + + Parameters + ---------- + ds : xarray.Dataset + dataset with model data. + gwf : flopy ModflowGwf + groundwaterflow object. + mask : str, optional + name of data variable in ds that is 1 for cells with a constant + head and zero for all other cells. The default is 'chd_mask'. + head : str, optional + name of data variable in ds that is used as the head in the chd + cells. By default, assumes head data is stored as 'chd_head'. + pname : str, optional + package name + auxiliary : str or list of str + name(s) of data arrays to include as auxiliary data to reclist + layer : int or None + The layer in which the boundary is added. It is added to the first active layer + when layer is None. The default is 0. + chd : str, optional + deprecated, the new argument is 'mask' + + Returns + ------- + chd : flopy ModflowGwfchd + chd package + """ + logger.info("creating mf6 CHD") + + if "chd" in kwargs: + warnings.warn( + "the 'chd' kwarg has been renamed to 'mask'!", + DeprecationWarning, + ) + mask = kwargs.pop("chd") + + maskarr = _get_value_from_ds_datavar(ds, "mask", mask, return_da=True) + mask = maskarr > 0 + + # get the stress_period_data + first_active_layer = layer is None + chd_rec = grid.da_to_reclist( + ds, + mask, + col1=head, + layer=layer, + aux=auxiliary, + first_active_layer=first_active_layer, + ) + + if len(chd_rec) > 0: + chd = flopy.mf6.ModflowGwfchd( + gwf, + auxiliary="CONCENTRATION" if auxiliary is not None else None, + pname=pname, + maxbound=len(chd_rec), + stress_period_data=chd_rec, + save_flows=True, + **kwargs, + ) + if (auxiliary is not None) and (ds.transport == 1): + logger.info("-> adding CHD to SSM sources list") + ssm_sources = list(ds.attrs["ssm_sources"]) + if chd.package_name not in ssm_sources: + ssm_sources += [chd.package_name] + ds.attrs["ssm_sources"] = ssm_sources + return chd + else: + logger.warning("no chd pkg added") + return None + + def ic(ds, gwf, starting_head="starting_head", pname="ic", **kwargs): - """get initial condictions package from model dataset. + """create initial condictions package from model dataset. Parameters ---------- @@ -520,11 +670,11 @@ def ic(ds, gwf, starting_head="starting_head", pname="ic", **kwargs): ic : flopy ModflowGwfic ic package """ - logger.info("creating modflow IC") + logger.info("creating mf6 IC") if isinstance(starting_head, numbers.Number): logger.info("adding 'starting_head' data array to ds") - ds["starting_head"] = starting_head * xr.ones_like(ds["idomain"]) + ds["starting_head"] = starting_head * xr.ones_like(ds["botm"]) ds["starting_head"].attrs["units"] = "mNAP" starting_head = "starting_head" @@ -537,14 +687,14 @@ def ic(ds, gwf, starting_head="starting_head", pname="ic", **kwargs): def sto( ds, gwf, - sy=0.2, - ss=0.000001, + sy="sy", + ss="ss", iconvert=1, save_flows=False, pname="sto", **kwargs, ): - """get storage package from model dataset. + """create storage package from model dataset. Parameters ---------- @@ -552,10 +702,10 @@ def sto( dataset with model data. gwf : flopy ModflowGwf groundwaterflow object. - sy : float, optional - specific yield. The default is 0.2. - ss : float, optional - specific storage. The default is 0.000001. + sy : str or float, optional + specific yield. The default is "sy", or 0.2 if "sy" is not in ds. + ss : str or float, optional + specific storage. The default is "ss", or 0.000001 if "ss" is not in ds. iconvert : int, optional See description in ModflowGwfsto. The default is 1 (differs from FloPY). save_flows : bool, optional @@ -569,20 +719,17 @@ def sto( sto : flopy ModflowGwfsto sto package """ - logger.info("creating modflow STO") + logger.info("creating mf6 STO") - if ds.time.steady_state: + if "time" not in ds or ds["steady"].all(): + logger.warning("Model is steady-state, no STO package created.") return None else: - if ds.time.steady_start: - sts_spd = {0: True} - trn_spd = {1: True} - else: - sts_spd = None - trn_spd = {0: True} + sts_spd = {iper: bool(b) for iper, b in enumerate(ds["steady"])} + trn_spd = {iper: not bool(b) for iper, b in enumerate(ds["steady"])} - sy = _get_value_from_ds_datavar(ds, "sy", sy) - ss = _get_value_from_ds_datavar(ds, "ss", ss) + sy = _get_value_from_ds_datavar(ds, "sy", sy, default=0.2) + ss = _get_value_from_ds_datavar(ds, "ss", ss, default=1e-5) sto = flopy.mf6.ModflowGwfsto( gwf, @@ -598,75 +745,8 @@ def sto( return sto -def chd( - ds, gwf, mask="chd_mask", head="chd_head", pname="chd", auxiliary=None, **kwargs -): - """get constant head boundary at the model's edges from the model dataset. - - Parameters - ---------- - ds : xarray.Dataset - dataset with model data. - gwf : flopy ModflowGwf - groundwaterflow object. - mask : str, optional - name of data variable in ds that is 1 for cells with a constant - head and zero for all other cells. The default is 'chd_mask'. - head : str, optional - name of data variable in ds that is used as the head in the chd - cells. By default, assumes head data is stored as 'chd_head'. - pname : str, optional - package name - auxiliary : str or list of str - name(s) of data arrays to include as auxiliary data to reclist - chd : str, optional - deprecated, the new argument is 'mask' - - Returns - ------- - chd : flopy ModflowGwfchd - chd package - """ - logger.info("creating modflow CHD") - - if "chd" in kwargs: - warnings.warn( - "the 'chd' kwarg has been renamed to 'mask'!", - DeprecationWarning, - ) - mask = kwargs.pop("chd") - - maskarr = _get_value_from_ds_datavar(ds, "mask", mask) - mask = maskarr != 0 - - # get the stress_period_data - chd_rec = grid.da_to_reclist(ds, mask, col1=head, aux=auxiliary) - - chd = flopy.mf6.ModflowGwfchd( - gwf, - auxiliary="CONCENTRATION" if auxiliary is not None else None, - pname=pname, - maxbound=len(chd_rec), - stress_period_data=chd_rec, - save_flows=True, - **kwargs, - ) - if (auxiliary is not None) and (ds.transport == 1): - logger.info("-> adding CHD to SSM sources list") - ssm_sources = ds.attrs["ssm_sources"] - if chd.package_name not in ssm_sources: - ssm_sources += [chd.package_name] - ds.attrs["ssm_sources"] = ssm_sources - - if len(chd_rec) > 0: - return chd - else: - logger.warning("no chd pkg added") - return None - - def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs): - """get surface level drain (maaivelddrainage in Dutch) from the model + """create surface level drain (maaivelddrainage in Dutch) from the model dataset. Parameters @@ -693,7 +773,7 @@ def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs ds.attrs["surface_drn_resistance"] = resistance - maskarr = _get_value_from_ds_datavar(ds, "elev", elev) + maskarr = _get_value_from_ds_datavar(ds, "elev", elev, return_da=True) mask = maskarr.notnull() drn_rec = grid.da_to_reclist( @@ -708,7 +788,6 @@ def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs drn = flopy.mf6.ModflowGwfdrn( gwf, pname=pname, - print_input=True, maxbound=len(drn_rec), stress_period_data={0: drn_rec}, save_flows=True, @@ -719,7 +798,7 @@ def surface_drain_from_ds(ds, gwf, resistance, elev="ahn", pname="drn", **kwargs def rch(ds, gwf, pname="rch", **kwargs): - """get recharge package from model dataset. + """create recharge package from model dataset. Parameters ---------- @@ -735,7 +814,7 @@ def rch(ds, gwf, pname="rch", **kwargs): rch : flopy ModflowGwfrch rch package """ - logger.info("creating modflow RCH") + logger.info("creating mf6 RCH") # create recharge package rch = recharge.ds_to_rch(gwf, ds, pname=pname, **kwargs) @@ -743,7 +822,7 @@ def rch(ds, gwf, pname="rch", **kwargs): def evt(ds, gwf, pname="evt", **kwargs): - """get evapotranspiration package from model dataset. + """create evapotranspiration package from model dataset. Parameters ---------- @@ -759,7 +838,7 @@ def evt(ds, gwf, pname="evt", **kwargs): evt : flopy ModflowGwfevt rch package """ - logger.info("creating modflow EVT") + logger.info("creating mf6 EVT") # create recharge package evt = recharge.ds_to_evt(gwf, ds, pname=pname, **kwargs) @@ -767,6 +846,31 @@ def evt(ds, gwf, pname="evt", **kwargs): return evt +def uzf(ds, gwf, pname="uzf", **kwargs): + """create unsaturated zone flow package from model dataset. + + Parameters + ---------- + ds : xarray.Dataset + dataset with model data. + gwf : flopy ModflowGwf + groundwaterflow object. + pname : str, optional + package name + + Returns + ------- + uzf : flopy ModflowGwfuzf + uzf package + """ + logger.info("creating mf6 UZF") + + # create uzf package + uzf = recharge.ds_to_uzf(gwf, ds, pname=pname, **kwargs) + + return uzf + + def _set_record(out, budget, output="head"): record = [] if isinstance(out, bool): @@ -847,7 +951,7 @@ def oc( pname="oc", **kwargs, ): - """get output control package from model dataset. + """create output control package from model dataset. Parameters ---------- @@ -863,7 +967,7 @@ def oc( oc : flopy ModflowGwfoc oc package """ - logger.info("creating modflow OC") + logger.info("creating mf6 OC") # Create the output control package headfile = f"{ds.model_name}.hds" diff --git a/nlmod/gwf/horizontal_flow_barrier.py b/nlmod/gwf/horizontal_flow_barrier.py index 7fd86d67..d6e9b557 100644 --- a/nlmod/gwf/horizontal_flow_barrier.py +++ b/nlmod/gwf/horizontal_flow_barrier.py @@ -118,7 +118,7 @@ def line2hfb(gdf, gwf, prevent_rings=True, plot=False): cellids : 2d list of ints a list with pairs of cells that have a hfb between them. """ - # for the idea, sea: + # for the idea, see: # https://gis.stackexchange.com/questions/188755/how-to-snap-a-road-network-to-a-hexagonal-grid-in-qgis gdfg = gdf_to_grid(gdf, gwf) @@ -240,7 +240,7 @@ def polygon_to_hfb( else: # find connections icvert = ds["icvert"].data - nodata = ds["icvert"].attrs["_FillValue"] + nodata = ds["icvert"].attrs["nodata"] edges = [] for icell2d in range(icvert.shape[0]): diff --git a/nlmod/gwf/lake.py b/nlmod/gwf/lake.py index af33dd2a..7fede9d1 100644 --- a/nlmod/gwf/lake.py +++ b/nlmod/gwf/lake.py @@ -48,7 +48,7 @@ def lake_from_gdf( Parameters ---------- - gwf : flopy.mf6.modflow.mfgwf.ModflowGwf + gwf : flopy.mf6.ModflowGwf groundwater flow model. gdf : gpd.GeoDataframe geodataframe with the cellids as the index and the columns: diff --git a/nlmod/gwf/output.py b/nlmod/gwf/output.py index 30955f2c..5b271b03 100644 --- a/nlmod/gwf/output.py +++ b/nlmod/gwf/output.py @@ -9,17 +9,71 @@ from shapely.geometry import Point from ..dims.grid import modelgrid_from_ds -from ..mfoutput import _get_output_da +from ..mfoutput.mfoutput import _get_budget_da, _get_heads_da, _get_time_index logger = logging.getLogger(__name__) -def get_heads_da(ds=None, gwf=None, fname_heads=None, fname_hds=None): - """Reads heads file given either a dataset or a groundwater flow object. +def get_headfile(ds=None, gwf=None, fname=None, grbfile=None): + """Get modflow HeadFile object. + + Provide one of ds, gwf or fname_hds. Not that it really matters but if + all are provided hierarchy is as follows: fname_hds > ds > gwf + + Parameters + ---------- + ds : xarray.Dataset, optional + model dataset, by default None + gwf : flopy.mf6.ModflowGwf, optional + groundwater flow model, by default None + fname : str, optional + path to heads file, by default None + grbfile : str + path to file containing binary grid information + + Returns + ------- + headobj : flopy.utils.HeadFile + HeadFile object handle + """ + msg = "Load the heads using either the ds, gwf or fname_hds" + assert ((ds is not None) + (gwf is not None) + (fname is not None)) >= 1, msg + + if fname is None: + if ds is None: + return gwf.output.head() + else: + fname = os.path.join(ds.model_ws, ds.model_name + ".hds") + # get grb file + if ds.gridtype == "vertex": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".disv.grb") + elif ds.gridtype == "structured": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".dis.grb") + else: + grbfile = None + + if fname is not None: + if grbfile is not None: + mg = flopy.mf6.utils.MfGrdFile(grbfile).modelgrid + else: + logger.warning(msg) + warnings.warn(msg) + mg = None + headobj = flopy.utils.HeadFile(fname, modelgrid=mg) + return headobj + + +def get_heads_da( + ds=None, + gwf=None, + fname=None, + grbfile=None, + delayed=False, + chunked=False, + **kwargs, +): + """Read binary heads file. - Note: Calling this function with ds is currently preferred over calling it - with gwf, because the layer and time coordinates can not be fully - reconstructed from gwf. Parameters ---------- @@ -27,29 +81,107 @@ def get_heads_da(ds=None, gwf=None, fname_heads=None, fname_hds=None): Xarray dataset with model data. gwf : flopy ModflowGwf Flopy groundwaterflow object. - fname_heads : path, optional - Instead of loading the binary heads file corresponding to ds or gwf - load the heads from - fname_hds : path, optional, Deprecated - please use fname_heads instead. + fname : path, optional + path to a binary heads file + grbfile : str, optional + path to file containing binary grid information, only needed if reading + output from file using fname + delayed : bool, optional + if delayed is True, do not load output data into memory, default is False. + chunked : bool, optional + chunk data array containing output, default is False. Returns ------- head_da : xarray.DataArray heads data array. """ - if fname_hds is not None: - logger.warning( - "Kwarg 'fname_hds' was renamed to 'fname_heads'. Please update your code." - ) - fname_heads = fname_hds - head_da = _get_output_da(_get_heads, ds=ds, gwf_or_gwt=gwf, fname=fname_heads) - head_da.attrs["units"] = "m NAP" - return head_da + hobj = get_headfile(ds=ds, gwf=gwf, fname=fname, grbfile=grbfile) + # gwf.output.head() defaults to a structured grid + if gwf is not None and ds is None and fname is None: + kwargs["modelgrid"] = gwf.modelgrid + da = _get_heads_da(hobj, **kwargs) + da.attrs["units"] = "m NAP" + + # set time index if ds/gwf are provided + if ds is not None or gwf is not None: + da["time"] = _get_time_index(hobj, ds=ds, gwf_or_gwt=gwf) + if ds is not None: + da["layer"] = ds.layer + + if chunked: + # chunk data array + da = da.chunk("auto") + + if not delayed: + # load into memory + da = da.compute() + + return da + + +def get_cellbudgetfile(ds=None, gwf=None, fname=None, grbfile=None): + """Get modflow CellBudgetFile object. + + Provide one of ds, gwf or fname_cbc. Not that it really matters but if + all are provided hierarchy is as follows: fname_cbc > ds > gwf + + Parameters + ---------- + ds : xarray.Dataset, optional + model dataset, by default None + gwf : flopy.mf6.ModflowGwf, optional + groundwater flow model, by default None + fname_cbc : str, optional + path to cell budget file, by default None\ + grbfile : str, optional + path to file containing binary grid information, only needed if + fname_cbc is passed as only argument. + + Returns + ------- + cbc : flopy.utils.CellBudgetFile + CellBudgetFile object + """ + msg = "Load the budgets using either the ds or the gwf" + assert ((ds is not None) + (gwf is not None) + (fname is not None)) == 1, msg + if fname is None: + if ds is None: + return gwf.output.budget() + else: + fname = os.path.join(ds.model_ws, ds.model_name + ".cbc") + # get grb file + if ds.gridtype == "vertex": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".disv.grb") + elif ds.gridtype == "structured": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".dis.grb") + else: + grbfile = None + if fname is not None: + if grbfile is not None: + mg = flopy.mf6.utils.MfGrdFile(grbfile).modelgrid + else: + logger.error("Cannot create budget data-array without grid information.") + raise ValueError( + "Please provide grid information by passing path to the " + "binary grid file with `grbfile=`." + ) + cbc = flopy.utils.CellBudgetFile(fname, modelgrid=mg) + return cbc -def get_budget_da(text, ds=None, gwf=None, fname_cbc=None, kstpkper=None): - """Reads budget file given either a dataset or a groundwater flow object. + +def get_budget_da( + text, + ds=None, + gwf=None, + fname=None, + grbfile=None, + delayed=False, + chunked=False, + **kwargs, +): + """Read binary budget file. Parameters ---------- @@ -59,56 +191,41 @@ def get_budget_da(text, ds=None, gwf=None, fname_cbc=None, kstpkper=None): xarray dataset with model data. One of ds or gwf must be provided. gwf : flopy ModflowGwf, optional Flopy groundwaterflow object. One of ds or gwf must be provided. - fname_cbc : path, optional + fname : path, optional specify the budget file to load, if not provided budget file will be obtained from ds or gwf. + grbfile : str + path to file containing binary grid information, only needed if reading + output from file using fname + delayed : bool, optional + if delayed is True, do not load output data into memory, default is False. + chunked : bool, optional + chunk data array containing output, default is False. Returns ------- - q_da : xarray.DataArray + da : xarray.DataArray budget data array. """ - q_da = _get_output_da( - _get_cbc, - ds=ds, - gwf_or_gwt=gwf, - fname=fname_cbc, - text=text, - kstpkper=kstpkper, - full3D=True, - ) - q_da.attrs["units"] = "m3/d" + cbcobj = get_cellbudgetfile(ds=ds, gwf=gwf, fname=fname, grbfile=grbfile) + da = _get_budget_da(cbcobj, text, **kwargs) + da.attrs["units"] = "m3/d" - return q_da + # set time index if ds/gwt are provided + if ds is not None or gwf is not None: + da["time"] = _get_time_index(cbcobj, ds=ds, gwf_or_gwt=gwf) + if ds is not None: + da["layer"] = ds.layer + if chunked: + # chunk data array + da = da.chunk("auto") -def _get_heads(ds=None, gwf=None, fname_hds=None): - msg = "Load the heads using either the ds, gwf or fname_hds" - assert ((ds is not None) + (gwf is not None) + (fname_hds is not None)) >= 1, msg + if not delayed: + # load into memory + da = da.compute() - if fname_hds is None: - if ds is None: - return gwf.output.head() - else: - fname_hds = os.path.join(ds.model_ws, ds.model_name + ".hds") - - headobj = flopy.utils.HeadFile(fname_hds) - - return headobj - - -def _get_cbc(ds=None, gwf=None, fname_cbc=None): - msg = "Load the budgets using either the ds or the gwf" - assert ((ds is not None) + (gwf is not None)) == 1, msg - - if fname_cbc is None: - if ds is None: - cbc = gwf.output.budget() - else: - fname_cbc = os.path.join(ds.model_ws, ds.model_name + ".cbc") - if fname_cbc is not None: - cbc = flopy.utils.CellBudgetFile(fname_cbc) - return cbc + return da def get_gwl_from_wet_cells(head, layer="layer", botm=None): @@ -188,7 +305,11 @@ def get_head_at_point(head, x, y, ds=None, gi=None, drop_nan_layers=True): if "icell2d" in head.dims: if gi is None: if ds is None: - raise (Exception("Please supply either gi or ds for a vertex grid")) + raise ( + ValueError( + "Please supply either gi (GridIntersect) or ds for a vertex grid" + ) + ) gi = flopy.utils.GridIntersect(modelgrid_from_ds(ds), method="vertex") icelld2 = gi.intersect(Point(x, y))["cellids"][0] head_point = head[:, :, icelld2] @@ -341,7 +462,7 @@ def calculate_gxg( >>> import nlmod >>> head = nlmod.gwf.get_heads_da(ds) - >>> gxg = nlmod.evaluate.calculate_gxg(head) + >>> gxg = nlmod.gwf.output.calculate_gxg(head) """ # if not head.dims == ("time", "y", "x"): # raise ValueError('Dimensions must be ("time", "y", "x")') diff --git a/nlmod/gwf/recharge.py b/nlmod/gwf/recharge.py index 574ce469..05feb150 100644 --- a/nlmod/gwf/recharge.py +++ b/nlmod/gwf/recharge.py @@ -1,20 +1,25 @@ -# -*- coding: utf-8 -*- -"""add knmi precipitation and evaporation to a modflow model.""" - - import logging import flopy import numpy as np -from tqdm import tqdm - -from ..dims.grid import da_to_reclist -from ..sim.sim import get_tdis_perioddata +import pandas as pd +import xarray as xr + +from ..dims.grid import cols_to_reclist, da_to_reclist +from ..dims.layers import ( + calculate_thickness, + get_first_active_layer_from_idomain, + get_idomain, +) +from ..dims.time import dataframe_to_flopy_timeseries +from ..util import _get_value_from_ds_datavar logger = logging.getLogger(__name__) -def ds_to_rch(gwf, ds, mask=None, pname="rch", **kwargs): +def ds_to_rch( + gwf, ds, mask=None, pname="rch", recharge="recharge", auxiliary=None, **kwargs +): """Convert the recharge data in the model dataset to a rch package with time series. @@ -28,10 +33,15 @@ def ds_to_rch(gwf, ds, mask=None, pname="rch", **kwargs): data array containing mask, recharge is only added where mask is True pname : str, optional package name. The default is 'rch'. + auxiliary : str or list of str + name(s) of data arrays to include as auxiliary data to reclist + recharge : str, optional + The name of the variable in ds that contains the recharge flux rate. The default + is "recharge". Returns ------- - rch : flopy.mf6.modflow.mfgwfrch.ModflowGwfrch + rch : flopy.mf6.ModflowGwfrch recharge package """ # check for nan values @@ -39,21 +49,28 @@ def ds_to_rch(gwf, ds, mask=None, pname="rch", **kwargs): raise ValueError("please remove nan values in recharge data array") # get stress period data - rch_name_arr, rch_unique_dic = _get_unique_series(ds, "recharge", pname) - ds["rch_name"] = ds["top"].dims, rch_name_arr - if mask is not None: - mask = (ds["rch_name"] != "") & mask + use_ts = "time" in ds[recharge].dims and len(ds["time"]) > 1 + if not use_ts: + recharge = ds[recharge] + if "time" in recharge.dims: + recharge = recharge.isel(time=0) + mask_recharge = recharge != 0 else: - mask = ds["rch_name"] != "" + rch_name_arr, rch_unique_dic = _get_unique_series(ds, recharge, pname) + ds["rch_name"] = ds["top"].dims, rch_name_arr + recharge = ds["rch_name"] + mask_recharge = recharge != "" - recharge = "rch_name" + if mask is not None: + mask_recharge = mask & mask_recharge spd = da_to_reclist( ds, - mask, + mask_recharge, col1=recharge, first_active_layer=True, only_active_cells=False, + aux=auxiliary, ) # create rch package @@ -62,18 +79,36 @@ def ds_to_rch(gwf, ds, mask=None, pname="rch", **kwargs): filename=f"{gwf.name}.rch", pname=pname, fixed_cell=False, + auxiliary="CONCENTRATION" if auxiliary is not None else None, maxbound=len(spd), stress_period_data={0: spd}, **kwargs, ) + if (auxiliary is not None) and (ds.transport == 1): + logger.info("-> adding GHB to SSM sources list") + ssm_sources = list(ds.attrs["ssm_sources"]) + if rch.package_name not in ssm_sources: + ssm_sources += [rch.package_name] + ds.attrs["ssm_sources"] = ssm_sources - # create timeseries packages - _add_time_series(rch, rch_unique_dic, ds) + if use_ts: + # create timeseries packages + _add_time_series(rch, rch_unique_dic, ds) return rch -def ds_to_evt(gwf, ds, pname="evt", nseg=1, surface=None, depth=None, **kwargs): +def ds_to_evt( + gwf, + ds, + mask=None, + pname="evt", + rate="evaporation", + nseg=1, + surface=None, + depth=None, + **kwargs, +): """Convert the evaporation data in the model dataset to a evt package with time series. @@ -83,8 +118,13 @@ def ds_to_evt(gwf, ds, pname="evt", nseg=1, surface=None, depth=None, **kwargs): groundwater flow model. ds : xr.DataSet dataset containing relevant model grid information + mask : xr.DataArray + data array containing mask, evt is only added where mask is True pname : str, optional package name. The default is 'evt'. + rate : str, optional + The name of the variable in ds that contains the maximum ET flux rate. The + default is "evaporation". nseg : int, optional number of ET segments. Only 1 is supported for now. The default is 1. surface : str, float or xr.DataArray, optional @@ -105,12 +145,12 @@ def ds_to_evt(gwf, ds, pname="evt", nseg=1, surface=None, depth=None, **kwargs): Returns ------- - evt : flopy.mf6.modflow.mfgwfevt.ModflowGwfevt + evt : flopy.mf6.ModflowGwfevt evapotranspiiration package. """ assert nseg == 1, "More than one evaporation segment not yet supported" if "surf_rate_specified" in kwargs: - raise (Exception("surf_rate_specified not yet supported")) + raise (NotImplementedError("surf_rate_specified not yet supported")) if surface is None: logger.info("Setting evaporation surface to 1 meter below top") surface = ds["top"] - 1.0 @@ -118,23 +158,29 @@ def ds_to_evt(gwf, ds, pname="evt", nseg=1, surface=None, depth=None, **kwargs): logger.info("Setting extinction depth to 1 meter below surface") depth = 1.0 - if ds["evaporation"].isnull().any(): + # check for nan values + if ds[rate].isnull().any(): raise ValueError("please remove nan values in evaporation data array") # get stress period data - if ds.time.steady_state: - mask = ds["evaporation"] != 0 - rate = "evaporation" + use_ts = "time" in ds[rate].dims and len(ds["time"]) > 1 + if not use_ts: + rate = ds[rate] + if "time" in rate.dims: + rate = rate.isel(time=0) + mask_rate = rate != 0 else: - evt_name_arr, evt_unique_dic = _get_unique_series(ds, "evaporation", pname) + evt_name_arr, evt_unique_dic = _get_unique_series(ds, rate, pname) ds["evt_name"] = ds["top"].dims, evt_name_arr + rate = ds["evt_name"] + mask_rate = rate != "" - mask = ds["evt_name"] != "" - rate = "evt_name" + if mask is not None: + mask_rate = mask & mask_rate spd = da_to_reclist( ds, - mask, + mask_rate, col1=surface, col2=rate, col3=depth, @@ -154,15 +200,318 @@ def ds_to_evt(gwf, ds, pname="evt", nseg=1, surface=None, depth=None, **kwargs): **kwargs, ) - if ds.time.steady_state: - return evt - - # create timeseries packages - _add_time_series(evt, evt_unique_dic, ds) + if use_ts: + # create timeseries packages + _add_time_series(evt, evt_unique_dic, ds) return evt +def ds_to_uzf( + gwf, + ds, + mask=None, + pname="uzf", + surfdep=0.05, + vks="kv", + thtr=0.1, + thts=0.3, + thti=0.1, + eps=3.5, + landflag=None, + finf="recharge", + pet="evaporation", + extdp=None, + extwc=None, + ha=None, + hroot=None, + rootact=None, + simulate_et=True, + linear_gwet=True, + unsat_etwc=False, + unsat_etae=False, + obs_depth_interval=None, + obs_z=None, + **kwargs, +): + """Create a unsaturated zone flow package for modflow 6. This method adds uzf-cells + to all active Modflow cells (unless mask is specified). + + Parameters + ---------- + gwf : flopy.mf6.modflow.mfgwf.ModflowGwf + groundwater flow model. + ds : xr.DataSet + dataset containing relevant model grid information + mask : xr.DataArray + data array containing mask, uzf is only added where mask is True + pname : str, optional + package name. The default is 'uzf'. + surfdep : float, str or array-like + surfdep is the surface depression depth of the UZF cell. When passed as string, + the array is obtained from ds. The default is 0.05 m. + vks : float, str or array-like + vks is the saturated vertical hydraulic conductivity of the UZF cell. This value + is used with the Brooks-Corey function and the simulated water content to + calculate the partially saturated hydraulic conductivity. When passed as string, + the array is obtained from ds. The default is 'kv'. + thtr : float, str or array-like + thtr is the residual (irreducible) water content of the UZF cell. This residual + water is not available to plants and will not drain into underlying aquifer + cells. When passed as string, the array is obtained from ds. The default is + 0.1. + thts : float, str or array-like + thts is the saturated water content of the UZF cell. The values for saturated + and residual water content should be set in a manner that is consistent with the + specific yield value specified in the Storage Package. The saturated water + content must be greater than the residual content. When passed as string, the + array is obtained from ds. The default is 0.3. + thti : float, str or array-like + thti is the initial water content of the UZF cell. The value must be greater + than or equal to the residual water content and less than or equal to the + saturated water content. When passed as string, the array is obtained from ds. + The default is 0.1. + eps : float, str or array-like + eps is the exponent used in the Brooks-Corey function. The Brooks-Corey function + is used by UZF to calculated hydraulic conductivity under partially saturated + conditions as a function of water content and the user-specified saturated + hydraulic conductivity. Values must be between 3.5 and 14.0. When passed as + string, the array is obtained from ds. The default is 3.5. + landflag : xr.DataArray, optional + A DataArray with integer values, set to one for land surface cells indicating + that boundary conditions can be applied and data can be specified in the PERIOD + block. A value of 0 specifies a non-land surface cell. Landflag is set to one + for the most upper active layer in each vertical column (determined form ds) + when it is None. The default is None. + finf :float, str or array-like + The applied infiltration rate of the UZF cell (:math:`LT^{-1}`). When passed as + string, the array is obtained from ds. The default is "recharge". + pet : float, str or array-like + The potential evapotranspiration rate of the UZF cell and specified GWF cell. + Evapotranspiration is first removed from the unsaturated zone and any remaining + potential evapotranspiration is applied to the saturated zone. only + used if simulate_et=True. When passed as string, the array is obtained from ds. + The default is "evaporation". + extdp : float, optional + Value that defines the evapotranspiration extinction depth of the UZF cell, in + meters below the top of the model. Set to 2.0 meter when None. The default is + None. + extwc : float, optional + The evapotranspiration extinction water content of the UZF cell. Only used if + simulate_et=True and unsat_etwc=True. Set to thtr when None. The default is + None. + ha : float, optional + The air entry potential (head) of the UZF cell. Only used if simulate_et=True + and unsat_etae=True. Set to 0.0 when None. The default is None. + hroot : float, optional + The root potential (head) of the UZF cell. Only used if simulate_et=True and + unsat_etae=True. Set to 0.0 when None. The default is None. + rootact : float, optional + the root activity function of the UZF cell. ROOTACT is the length of roots in + a given volume of soil divided by that volume. Values range from 0 to about 3 + :math:`cm^{-2}`, depending on the plant community and its stage of development. + Only used if simulate_et=True and unsat_etae=True. Set to 0.0 when None. The + default is None. + simulate_et : bool, optional + If True, ET in the unsaturated (UZF) and saturated zones (GWF) will be + simulated. The default is True. + linear_gwet : bool, optional + If True, groundwater ET will be simulated using the original ET formulation of + MODFLOW-2005. When False, and square_gwet is True as an extra argument, no + evaporation from the saturated zone will be simulated. The default is True. + unsat_etwc : bool, optional + If True, ET in the unsaturated zone will be simulated as a function of the + specified PET rate while the water content (THETA) is greater than the ET + extinction water content (EXTWC). The default is False. + unsat_etae : bool, optional + If True, ET in the unsaturated zone will be simulated using a capillary pressure + based formulation. Capillary pressure is calculated using the Brooks-Corey + retention function. The default is False. + obs_depth_interval : float, optional + The depths at which observations of the water content in each cell are added. + When not None, this creates a CSV output file with water content at different + z-coordinates in each UZF cell. The default is None. + obs_z : array-like, optional + The z-coordinate at which observations of the water content in each cell are + added. When not None, this creates a CSV output file with water content at fixes + z-coordinates in each UZF cell. The default is None. + ** kwargs : dict + Kwargs are passed onto flopy.mf6.ModflowGwfuzf + + + Returns + ------- + uzf : flopy.mf6.ModflowGwfuzf + Unsaturated zone flow package for Modflow 6. + """ + if mask is None: + mask = ds["area"] > 0 + + if "layer" not in mask.dims: + mask = mask.expand_dims(dim={"layer": ds.layer}) + + # only add uzf-cells in active cells + idomain = get_idomain(ds) + mask = mask & (idomain > 0) + + # generate packagedata + surfdep = _get_value_from_ds_datavar(ds, "surfdep", surfdep, return_da=True) + vks = _get_value_from_ds_datavar(ds, "vk", vks, return_da=True) + thtr = _get_value_from_ds_datavar(ds, "thtr", thtr, return_da=True) + thts = _get_value_from_ds_datavar(ds, "thts", thts, return_da=True) + thti = _get_value_from_ds_datavar(ds, "thti", thti, return_da=True) + eps = _get_value_from_ds_datavar(ds, "eps", eps, return_da=True) + + nuzfcells = int(mask.sum()) + cellids = np.where(mask) + + iuzno = xr.full_like(ds["botm"], -1, dtype=int) + iuzno.data[mask] = np.arange(nuzfcells) + + if landflag is None: + landflag = xr.full_like(ds["botm"], 0, dtype=int) + # set the landflag in the top layer to 1 + landflag[get_first_active_layer_from_idomain(idomain)] = 1 + + # determine ivertcon, by setting its value to iuzno of the layer below + ivertcon = xr.full_like(ds["botm"], -1, dtype=int) + ivertcon.data[:-1] = iuzno.data[1:] + # then use bfill to accont for inactive cells in the layer below, and set nans to -1 + ivertcon = ivertcon.where(ivertcon >= 0).bfill("layer").fillna(-1).astype(int) + + # packagedata : [iuzno, cellid, landflag, ivertcon, surfdep, vks, thtr, thts, thti, eps, boundname] + packagedata = cols_to_reclist( + ds, + cellids, + iuzno, + landflag, + ivertcon, + surfdep, + vks, + thtr, + thts, + thti, + eps, + cellid_column=1, + ) + + # add perioddata for all uzf cells that are at the surface + mask = landflag == 1 + + # perioddata : [iuzno, finf, pet, extdp, extwc, ha, hroot, rootact, aux] + finf_name_arr, uzf_unique_dic = _get_unique_series(ds, finf, "finf") + finf = "rch_name" + ds[finf] = ds["top"].dims, finf_name_arr + ds[finf] = ds[finf].expand_dims(dim={"layer": ds.layer}) + if mask is not None: + mask = (ds[finf] != "") & mask + else: + mask = ds[finf] != "" + + pet_name_arr, pet_unique_dic = _get_unique_series(ds, pet, "pet") + pet = "evt_name" + ds[pet] = ds["top"].dims, pet_name_arr + ds[pet] = ds[pet].expand_dims(dim={"layer": ds.layer}) + if mask is not None: + mask = (ds[pet] != "") & mask + else: + mask = ds[pet] != "" + + # combine the time series of finf and pet + uzf_unique_dic.update(pet_unique_dic) + + if extdp is None: + extdp = 2.0 + # EXTDP is always specified, but is only used if SIMULATE_ET + if simulate_et: + logger.info(f"Setting extinction depth (extdp) to {extdp} meter below top") + if extwc is None: + extwc = thtr + if simulate_et and unsat_etwc: + logger.info( + f"Setting evapotranspiration extinction water content (extwc) to {extwc}" + ) + if ha is None: + ha = 0.0 + if simulate_et and unsat_etae: + logger.info(f"Setting air entry potential (ha) to {ha}") + if hroot is None: + hroot = 0.0 + if simulate_et and unsat_etae: + logger.info(f"Setting root potential (hroot) to {hroot}") + if rootact is None: + rootact = 0.0 + if simulate_et and unsat_etae: + logger.info(f"Setting root activity function (rootact) to {rootact}") + + cellids_land = np.where(mask) + + perioddata = cols_to_reclist( + ds, + cellids_land, + iuzno, + finf, + pet, + extdp, + extwc, + ha, + hroot, + rootact, + cellid_column=None, + ) + + observations = None + # observation nodes uzf + if obs_depth_interval is not None or obs_z is not None: + cellid_per_iuzno = list(zip(*cellids)) + cellid_str = [ + str(x).replace("(", "").replace(")", "").replace(", ", "_") + for x in cellid_per_iuzno + ] + thickness = calculate_thickness(ds).data[iuzno >= 0] + # account for surfdep, as this decreases the height of the top of the upper cell + # otherwise modflow may return an error + thickness = thickness - landflag.data[iuzno >= 0] * surfdep / 2 + + obsdepths = [] + if obs_depth_interval is not None: + for i in range(nuzfcells): + depths = np.arange(obs_depth_interval, thickness[i], obs_depth_interval) + for depth in depths: + name = f"wc_{cellid_str[i]}_{depth:0.2f}" + obsdepths.append((name, "water-content", i + 1, depth)) + if obs_z is not None: + botm = ds["botm"].data[iuzno >= 0] + + top = botm + thickness + for i in range(nuzfcells): + mask = (obs_z > botm[i]) & (obs_z <= top[i]) + for z in obs_z[mask]: + depth = top[i] - z + name = f"wc_{cellid_str[i]}_{z:0.2f}" + obsdepths.append((name, "water-content", i + 1, depth)) + + observations = {ds.model_name + ".uzf.obs.csv": obsdepths} + + uzf = flopy.mf6.ModflowGwfuzf( + gwf, + filename=f"{gwf.name}.uzf", + pname=pname, + nuzfcells=nuzfcells, + packagedata=packagedata, + perioddata={0: perioddata}, + simulate_et=simulate_et, + linear_gwet=linear_gwet, + unsat_etwc=unsat_etwc, + unsat_etae=unsat_etae, + observations=observations, + **kwargs, + ) + + # create timeseries packages + _add_time_series(uzf, uzf_unique_dic, ds) + + def _get_unique_series(ds, var, pname): """Get the location and values of unique time series from a variable var in ds. @@ -238,31 +587,16 @@ def _add_time_series(package, rch_unique_dic, ds): ------- None. """ - # get timesteps - tdis_perioddata = get_tdis_perioddata(ds) - perlen_arr = [t[0] for t in tdis_perioddata] - time_steps_rch = [0.0] + np.array(perlen_arr).cumsum().tolist() - - for i, key in tqdm( - enumerate(rch_unique_dic.keys()), - total=len(rch_unique_dic.keys()), - desc="Building ts packages rch", - ): - # add extra time step to the time series object (otherwise flopy fails) - recharge_val = list(rch_unique_dic[key]) + [0.0] - - recharge = list(zip(time_steps_rch, recharge_val)) - if i == 0: - package.ts.initialize( - filename=f"{key}.ts", - timeseries=recharge, - time_series_namerecord=key, - interpolation_methodrecord="stepwise", - ) - else: - package.ts.append_package( - filename=f"{key}.ts", - timeseries=recharge, - time_series_namerecord=key, - interpolation_methodrecord="stepwise", - ) + # generate a DataFrame + df = pd.DataFrame(rch_unique_dic, index=ds.time) + if df.isna().any(axis=None): + # make sure there are no NaN's, as otherwise they will be filled by zeros later + raise (ValueError("There cannot be nan's in the DataFrame")) + # set the first value for the start-time as well + df.loc[pd.to_datetime(ds.time.start)] = df.iloc[0] + # combine the values with the start of each period, and set the last value to 0.0 + df = df.sort_index().shift(-1).fillna(0.0) + + dataframe_to_flopy_timeseries( + df, ds=ds, package=package, interpolation_methodrecord="stepwise" + ) diff --git a/nlmod/gwf/surface_water.py b/nlmod/gwf/surface_water.py index ba3828e6..37eed7c2 100644 --- a/nlmod/gwf/surface_water.py +++ b/nlmod/gwf/surface_water.py @@ -1,5 +1,6 @@ import logging import warnings +from functools import partial import flopy import numpy as np @@ -10,6 +11,7 @@ from tqdm import tqdm from ..dims.grid import gdf_to_grid +from ..dims.layers import get_idomain from ..dims.resample import get_extent_polygon from ..read import bgt, waterboard @@ -382,7 +384,7 @@ def build_spd( top = ds.top.data botm = ds.botm.data - idomain = ds.idomain.data + idomain = get_idomain(ds).data kh = ds.kh.data # ignore records without a stage @@ -419,7 +421,7 @@ def build_spd( # stage stage = row["stage"] - if (stage < rbot) and np.isfinite(rbot): + if not isinstance(stage, str) and stage < rbot and np.isfinite(rbot): logger.warning( f"WARNING: stage below bottom elevation in {cellid}, " "stage reset to rbot!" @@ -461,12 +463,12 @@ def build_spd( mask = (stage > botm_cell) & (idomain_cell > 0) if not mask.any(): raise ( - Exception("rbot and stage are below the bottom of the model") + ValueError("rbot and stage are below the bottom of the model") ) lays = [np.where(mask)[0][0]] conds = [cond] else: - raise (Exception(f"Method {layer_method} unknown")) + raise (ValueError(f"Method {layer_method} unknown")) auxlist = [] if "aux" in row: @@ -508,7 +510,7 @@ def add_info_to_gdf( inds = s.query(geom_to) if len(inds) == 0: continue - overlap = gdf_from.geometry[inds].intersection(geom_to) + overlap = gdf_from.geometry.iloc[inds].intersection(geom_to) if geom_type is None: geom_type = overlap.geom_type.iloc[0] if geom_type in ["Polygon", "MultiPolygon"]: @@ -519,7 +521,7 @@ def add_info_to_gdf( measure = overlap.length else: msg = f"Unsupported geometry type: {geom_type}" - raise (Exception(msg)) + raise TypeError(msg) if np.any(measure.sum() > min_total_overlap * measure_org): # take the largest @@ -806,7 +808,7 @@ def get_gdf(ds=None, extent=None, fname_ahn=None, ahn=None, buffer=0.0): """ if extent is None: if ds is None: - raise (Exception("Please supply either ds or extent to get_gdf")) + raise (ValueError("Please supply either ds or extent to get_gdf")) extent = get_extent_polygon(ds) gdf = bgt.get_bgt(extent) if fname_ahn is not None: @@ -851,7 +853,6 @@ def add_min_ahn_to_gdf(gdf, ahn, buffer=0.0, column="ahn_min"): A GeoDataFrame with surface water features, with an added column containing the minimum surface level height near the features. """ - from functools import partial from geocube.api.core import make_geocube from geocube.rasterize import rasterize_image @@ -906,7 +907,7 @@ def gdf_to_seasonal_pkg( The default water depth, only used when there is no 'rbot' column in gdf or when this column contains nans. The default is 0.5. boundname_column : str, optional - THe name of the column in gdf to use for the boundnames. The default is + The name of the column in gdf to use for the boundnames. The default is "identificatie", which is a unique identifier in the BGT. c0 : float, optional The resistance of the surface water, in days. Only used when there is @@ -989,7 +990,9 @@ def gdf_to_seasonal_pkg( spd[:, [2, 3]] = spd[:, [3, 2]] spd = spd.tolist() - if boundname_column is not None: + if boundname_column is None: + observations = None + else: observations = [] for boundname in np.unique(gdf[boundname_column]): observations.append((boundname, pkg, boundname)) @@ -1001,7 +1004,7 @@ def gdf_to_seasonal_pkg( elif pkg == "GHB": cl = flopy.mf6.ModflowGwfghb else: - raise (Exception(f"Unknown package: {pkg}")) + raise (ValueError(f"Unknown package: {pkg}")) package = cl( gwf, stress_period_data={0: spd}, @@ -1012,7 +1015,13 @@ def gdf_to_seasonal_pkg( **kwargs, ) # add timeseries for the seasons 'winter' and 'summer' - add_season_timeseries(ds, package, summer_months=summer_months, seasons=seasons) + add_season_timeseries( + ds, + package, + summer_months=summer_months, + winter_name="winter", + summer_name="summer", + ) return package @@ -1021,8 +1030,26 @@ def add_season_timeseries( package, summer_months=(4, 5, 6, 7, 8, 9), filename="season.ts", - seasons=("winter", "summer"), + winter_name="winter", + summer_name="summer", ): + """Add time series indicating which season is active (e.g. summer/winter). + + Parameters + ---------- + ds : xarray.Dataset + xarray dataset used for time discretization + package : flopy.mf6 package + Modflow 6 package to add time series to + summer_months : tuple, optional + summer months. The default is (4, 5, 6, 7, 8, 9), so from april to september. + filename : str, optional + name of time series file. The default is "season.ts". + winter_name : str, optional + The name of the time-series with ones in winter. The default is "winter". + summer_name : str, optional + The name of the time-series with ones in summer. The default is "summer". + """ tmin = pd.to_datetime(ds.time.start) if tmin.month in summer_months: ts_data = [(0.0, 0.0, 1.0)] @@ -1042,23 +1069,17 @@ def add_season_timeseries( if time > 0: ts_data.append((time, 1.0, 0.0)) - package.ts.initialize( + return package.ts.initialize( filename=filename, timeseries=ts_data, - time_series_namerecord=seasons, + time_series_namerecord=[winter_name, summer_name], interpolation_methodrecord=["stepwise", "stepwise"], ) def rivdata_from_xylist(gwf, xylist, layer, stage, cond, rbot): - # TODO: temporary fix until flopy is patched - if gwf.modelgrid.grid_type == "structured": - gi = flopy.utils.GridIntersect(gwf.modelgrid, rtree=False) - cellids = gi.intersect(xylist, shapetype="linestring")["cellids"] - else: - gi = flopy.utils.GridIntersect(gwf.modelgrid) - cellids = gi.intersects(xylist, shapetype="linestring")["cellids"] - + gi = flopy.utils.GridIntersect(gwf.modelgrid, method="vertex") + cellids = gi.intersect(xylist, shapetype="linestring")["cellids"] riv_data = [] for cid in cellids: if len(cid) == 2: diff --git a/nlmod/gwf/wells.py b/nlmod/gwf/wells.py index f958cfb6..3c8f0ced 100644 --- a/nlmod/gwf/wells.py +++ b/nlmod/gwf/wells.py @@ -1,6 +1,13 @@ +import logging + import flopy as fp +import geopandas as gpd import numpy as np -from tqdm import tqdm +import pandas as pd + +from ..dims.grid import gdf_to_grid + +logger = logging.getLogger(__name__) def wel_from_df( @@ -13,51 +20,104 @@ def wel_from_df( Q="Q", aux=None, boundnames=None, + ds=None, + auxmultname="multiplier", **kwargs, ): + """ + Add a Well (WEL) package based on input from a (Geo)DataFrame. + + Parameters + ---------- + df : pd.DataFrame or gpd.GeoDataFrame + A (Geo)DataFrame containing the properties of the wells. + gwf : flopy ModflowGwf + Groundwaterflow object to add the wel-package to. + x : str, optional + The column in df that contains the x-coordinate of the well. Only used when df + is a DataFrame. The default is 'x'. + y : str, optional + The column in df that contains the y-coordinate of the well. Only used when df + is a DataFrame. The default is 'y'. + top : str + The column in df that contains the z-coordinate of the top of the well screen. + The defaults is 'top'. + botm : str + The column in df that contains the z-coordinate of the bottom of the well + screen. The defaults is 'botm'. + Q : str, optional + The column in df that contains the volumetric well rate. This column can contain + floats, or strings belonging to timeseries added later. A positive value + indicates recharge (injection) and a negative value indicates discharge + (extraction) The default is "Q". + aux : str of list of str, optional + The column(s) in df that contain auxiliary variables. The default is None. + boundnames : str, optional + THe column in df thet . The default is None. + ds : xarray.Dataset + Dataset with model data. Needed to determine cellid when grid-rotation is used. + The default is None. + auxmultname : str, optional + The name of the auxiliary varibale that contains the multiplication factors to + distribute the well discharge over different layers. When auxmultname is None, + this auxiliary variable will not be added, and Q is multiplied by these factors + directly. auxmultname cannot be None when df[Q] contains strings (the names of + timeseries). The default is "multiplier". + **kwargs : dict + kwargs are passed to flopy.mf6.ModflowGwfwel. + + Returns + ------- + wel : flopy.mf6.ModflowGwfwel + wel package. + + """ + if aux is None: + aux = [] + if not isinstance(aux, list): + aux = [aux] + + df = _add_cellid(df, ds=ds, gwf=gwf, x=x, y=y) + multipliers = _get_layer_multiplier_for_wells(df, top, botm, ds=ds, gwf=gwf) + # collect data well_lrcd = [] - - for wnam, irow in tqdm(df.iterrows(), total=df.index.size, desc="Adding WELs"): - try: - cid1 = gwf.modelgrid.intersect(irow[x], irow[y], irow[top], forgive=False) - cid2 = gwf.modelgrid.intersect(irow[x], irow[y], irow[botm], forgive=False) - except Exception: - print( - f"Warning! well {wnam} outside of model domain! ({irow[x]}, {irow[y]})" - ) - continue - kb = cid2[0] - if len(cid1) == 2: - kt, icell2d = cid1 - idomain_mask = gwf.modelgrid.idomain[kt : kb + 1, icell2d] > 0 - elif len(cid1) == 3: - kt, i, j = cid1 - idomain_mask = gwf.modelgrid.idomain[kt : kb + 1, i, j] > 0 - # mask only active cells - wlayers = np.arange(kt, kb + 1)[idomain_mask] + for index, irow in df.iterrows(): + wlayers = np.where(multipliers[index] > 0)[0] for k in wlayers: - if len(cid1) == 2: - wdata = [(k, icell2d), irow[Q] / len(wlayers)] - elif len(cid1) == 3: - wdata = [(k, i, j), irow[Q] / len(wlayers)] - - if aux is not None: - if not isinstance(aux, list): - aux = [aux] - for iaux in aux: - wdata.append(irow[iaux]) + multiplier = multipliers[index][k] + q = irow[Q] + if auxmultname is None: + q = q * multiplier + if isinstance(irow["cellid"], int): + # vertex grid + cellid = (k, irow["cellid"]) + else: + # structured grid + cellid = (k, irow["cellid"][0], irow["cellid"][1]) + wdata = [cellid, q] + for iaux in aux: + wdata.append(irow[iaux]) + if auxmultname is not None: + wdata.append(multiplier) if boundnames is not None: wdata.append(irow[boundnames]) well_lrcd.append(wdata) + if auxmultname is not None: + aux.append(auxmultname) + wel_spd = {0: well_lrcd} + if len(aux) == 0: + aux = None + wel = fp.mf6.ModflowGwfwel( gwf, stress_period_data=wel_spd, auxiliary=aux, boundnames=boundnames is not None, + auxmultname=auxmultname, **kwargs, ) @@ -75,75 +135,272 @@ def maw_from_df( rw="rw", condeqn="THIEM", strt=0.0, + aux=None, boundnames=None, + ds=None, **kwargs, ): - maw_pakdata = [] - maw_conndata = [] - maw_perdata = [] - - for iw, irow in tqdm(df.iterrows(), total=df.index.size, desc="Adding MAWs"): - try: - cid1 = gwf.modelgrid.intersect(irow[x], irow[y], irow[top], forgive=False) - cid2 = gwf.modelgrid.intersect(irow[x], irow[y], irow[botm], forgive=False) - except Exception: - print(f"Warning! well {iw} outside of model domain! ({irow[x]}, {irow[y]})") - continue - kb = cid2[0] - if len(cid1) == 2: - kt, icell2d = cid1 - idomain_mask = gwf.modelgrid.idomain[kt : kb + 1, icell2d] > 0 - elif len(cid1) == 3: - kt, i, j = cid1 - idomain_mask = gwf.modelgrid.idomain[kt : kb + 1, i, j] > 0 - - wlayers = np.arange(kt, kb + 1)[idomain_mask] - - # - pakdata = [iw, irow[rw], irow[top], strt, condeqn, len(wlayers)] + """ + Add a Multi-AquiferWell (MAW) package based on input from a (Geo)DataFrame. + + Parameters + ---------- + df : pd.DataFrame or gpd.GeoDataFrame + A (Geo)DataFrame containing the properties of the wells. + gwf : flopy ModflowGwf + Groundwaterflow object to add the wel-package to. + x : str, optional + The column in df that contains the x-coordinate of the well. Only used when df + is a DataFrame. The default is 'x'. + y : str, optional + The column in df that contains the y-coordinate of the well. Only used when df + is a DataFrame. The default is 'y'. + top : str + The column in df that contains the z-coordinate of the top of the well screen. + The defaults is 'top'. + botm : str + The column in df that contains the z-coordinate of the bottom of the well + screen. The defaults is 'botm'. + Q : str, optional + The column in df that contains the volumetric well rate. This column can contain + floats, or strings belonging to timeseries added later. A positive value + indicates recharge (injection) and a negative value indicates discharge + (extraction) The default is "Q". + rw : str, optional + The column in df that contains the radius for the multi-aquifer well. The + default is "rw". + condeqn : str, optional + String that defines the conductance equation that is used to calculate the + saturated conductance for the multi-aquifer well. The default is "THIEM". + strt : float, optional + The starting head for the multi-aquifer well. The default is 0.0. + aux : str of list of str, optional + The column(s) in df that contain auxiliary variables. The default is None. + boundnames : str, optional + THe column in df thet . The default is None. + ds : xarray.Dataset + Dataset with model data. Needed to determine cellid when grid-rotation is used. + The default is None. + **kwargs : TYPE + Kwargs are passed onto ModflowGwfmaw. + + Returns + ------- + wel : flopy.mf6.ModflowGwfmaw + maw package. + + """ + if aux is None: + aux = [] + if not isinstance(aux, list): + aux = [aux] + + df = _add_cellid(df, ds=ds, gwf=gwf, x=x, y=y) + multipliers = _get_layer_multiplier_for_wells(df, top, botm, ds=ds, gwf=gwf) + + packagedata = [] + connectiondata = [] + perioddata = [] + + iw = 0 + for index, irow in df.iterrows(): + wlayers = np.where(multipliers[index] > 0)[0] + + # [wellno, radius, bottom, strt, condeqn, ngwfnodes] + pakdata = [iw, irow[rw], irow[botm], strt, condeqn, len(wlayers)] + for iaux in aux: + pakdata.append(irow[iaux]) if boundnames is not None: pakdata.append(irow[boundnames]) - maw_pakdata.append(pakdata) - # - maw_perdata.append([iw, "RATE", irow[Q]]) + packagedata.append(pakdata) + + # [wellno mawsetting] + perioddata.append([iw, "RATE", irow[Q]]) for iwellpart, k in enumerate(wlayers): - if gwf.modelgrid.grid_type == "vertex": - laytop = gwf.modelgrid.botm[k - 1, icell2d] - laybot = gwf.modelgrid.botm[k, icell2d] - # - mawdata = [ - iw, - iwellpart, - (k, icell2d), - np.min([irow[top], laytop]), - laybot, - 0.0, - 0.0, - ] - elif gwf.modelgrid.grid_type == "structured": - laytop = gwf.modelgrid.botm[k - 1, i, j] - laybot = gwf.modelgrid.botm[k, i, j] - # - mawdata = [ - iw, - iwellpart, - (k, i, j), - np.min([irow[top], laytop]), - laybot, - 0.0, - 0.0, - ] - maw_conndata.append(mawdata) + if k == 0: + laytop = gwf.modelgrid.top + else: + laytop = gwf.modelgrid.botm[k - 1] + laybot = gwf.modelgrid.botm[k] + + if isinstance(irow["cellid"], int): + # vertex grid + cellid = (k, irow["cellid"]) + laytop = laytop[irow["cellid"]] + laybot = laybot[irow["cellid"]] + else: + # structured grid + cellid = (k, irow["cellid"][0], irow["cellid"][1]) + laytop = laytop[irow["cellid"][0], irow["cellid"][1]] + laybot = laybot[irow["cellid"][0], irow["cellid"][1]] + scrn_top = np.min([irow[top], laytop]) + scrn_bot = np.max([irow[botm], laybot]) + # [wellno, icon, cellid, scrn_top, scrn_bot, hk_skin, radius_skin] + condata = [ + iw, + iwellpart, + cellid, + scrn_top, + scrn_bot, + 0.0, + 0.0, + ] + connectiondata.append(condata) + iw += 1 + + if len(aux) == 0: + aux = None maw = fp.mf6.ModflowGwfmaw( gwf, - nmawwells=df.index.size, + nmawwells=iw, + auxiliary=aux, boundnames=boundnames is not None, - packagedata=maw_pakdata, - connectiondata=maw_conndata, - perioddata=maw_perdata, + packagedata=packagedata, + connectiondata=connectiondata, + perioddata=perioddata, **kwargs, ) return maw + + +def _add_cellid(df, ds=None, gwf=None, x="x", y="y"): + """ + Intersect a DataFrame of point Data with the model grid, and add cellid-column. + + Parameters + ---------- + df : pd.DataFrame or gpd.GeoDataFrame + A (Geo)DataFrame containing the properties of the wells. + ds : xarray.Dataset + Dataset with model data. Either supply ds or gwf. The default is None. + gwf : flopy ModflowGwf + Groundwaterflow object. Only used when ds is None. The default is None. + x : str, optional + The column in df that contains the x-coordinate of the well. Only used when df + is a DataFrame. The default is 'x'. + y : str, optional + The column in df that contains the y-coordinate of the well. Only used when df + is a DataFrame. The default is 'y'. + + Returns + ------- + df : gpd.GeoDataFrame + A GeoDataFrame with a column named cellid that contains the icell2d-number + (vertex-grid) or (row, column) (structured grid). + + """ + if not isinstance(df, gpd.GeoDataFrame): + df = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df[x], df[y])) + if "cellid" not in df.columns: + df = gdf_to_grid(df, gwf if ds is None else ds) + return df + + +def _get_layer_multiplier_for_wells(df, top, botm, ds=None, gwf=None): + """ + Get factors (pandas.DataFrame) for each layer that well screens intersects with. + + Parameters + ---------- + df : pd.DataFrame + A DataFrame containing the properties of the wells. + top : str + The column in df that contains the z-coordinate of the top of the well screen. + botm : str + The column in df that contains the z-coordinate of the bottom of the well + screen. + ds : xarray.Dataset + Dataset with model data. Either supply ds or gwf. The default is None. + gwf : flopy ModflowGwf + Groundwaterflow object. Only used when ds is None. The default is None. + + Returns + ------- + multipliers : pd.DataFrame + A DataFrame containg the multiplication factors, with the layers as the index + and the name of the well screens (the index of df) as columns. + + """ + # get required data either from gwf or ds + if ds is not None: + ml_top = ds["top"].data + ml_bot = ds["botm"].data + kh = ds["kh"].data + layer = ds.layer + elif gwf is not None: + ml_top = gwf.dis.top.array + ml_bot = gwf.dis.botm.array + kh = gwf.npf.k.array + layer = range(gwf.dis.nlay.array) + else: + raise (TypeError("Either supply ds or gwf to determine layer multiplyer")) + + multipliers = {} + for index, irow in df.iterrows(): + multipliers[index] = _get_layer_multiplier_for_well( + irow["cellid"], irow[top], irow[botm], ml_top, ml_bot, kh + ) + + if (multipliers[index] == 0).all(): + logger.warning(f"No layers found for well {index}") + multipliers = pd.DataFrame(multipliers, index=layer, columns=df.index) + return multipliers + + +def _get_layer_multiplier_for_well(cid, well_top, well_bot, ml_top, ml_bot, ml_kh): + """ + Get a factor (numpy array) for each layer that a well screen intersects with. + + Parameters + ---------- + cid : int or tuple of 2 ints + THe cellid of the well (either icell2d or (row, column). + well_top : float + The z-coordinate of the top of the well screen. + well_bot : float + The z-coordinate of the top of the well screen. + ml_top : numpy array + The top of the upper layer of the model (1d or 2d) + ml_bot : numpy array + The bottom of all cells of the model (2d or 3d) + ml_kh : numpy array + The horizontal conductivity of all cells of the model (2d or 3d). + + Returns + ------- + multiplier : numpy array + An array with a factor (between 0 and 1) for each of the model layers. + + """ + # keep the tops and botms of the cell where the well is in + ml_top_cid = ml_top[cid].copy() + if isinstance(cid, int): + ml_bot_cid = ml_bot[:, cid].copy() + ml_kh_cid = ml_kh[:, cid].copy() + else: + ml_bot_cid = ml_bot[:, cid[0], cid[1]].copy() + ml_kh_cid = ml_kh[:, cid[0], cid[1]].copy() + ml_top_cid = np.array([ml_top_cid] + list(ml_bot_cid[:-1])) + + # only keep the part of layers along the well filter + ml_top_cid[ml_top_cid > well_top] = well_top + ml_top_cid[ml_top_cid < well_bot] = well_bot + ml_bot_cid[ml_bot_cid > well_top] = well_top + ml_bot_cid[ml_bot_cid < well_bot] = well_bot + + # calculate remaining kd along the well filter + kd = ml_kh_cid * (ml_top_cid - ml_bot_cid) + mask = kd < 0 + if np.any(mask): + logger.warning("There are negative thicknesses at cellid {cid}") + kd[mask] = 0 + if (kd == 0).all(): + # the well does not cross any of the layers. Just return an array of zeros. + multiplier = kd + else: + # divide by the total kd to get a factor + multiplier = kd / kd.sum() + return multiplier diff --git a/nlmod/gwt/gwt.py b/nlmod/gwt/gwt.py index aa1f98e8..7311715a 100644 --- a/nlmod/gwt/gwt.py +++ b/nlmod/gwt/gwt.py @@ -31,7 +31,7 @@ def gwt(ds, sim, modelname=None, **kwargs): """ # start creating model - logger.info("creating modflow GWT") + logger.info("creating mf6 GWT") # Create the Flopy groundwater flow (gwf) model object if modelname is None: @@ -107,7 +107,7 @@ def adv(ds, gwt, scheme=None, **kwargs): adv : flopy ModflowGwtadv adv package """ - logger.info("creating modflow ADV") + logger.info("creating mf6 ADV") scheme = _get_value_from_ds_attr(ds, "scheme", "adv_scheme", value=scheme) adv = flopy.mf6.ModflowGwtadv(gwt, scheme=scheme, **kwargs) return adv @@ -128,7 +128,7 @@ def dsp(ds, gwt, **kwargs): dsp : flopy ModflowGwtdsp dsp package """ - logger.info("creating modflow DSP") + logger.info("creating mf6 DSP") alh = _get_value_from_ds_attr(ds, "alh", "dsp_alh", value=kwargs.pop("alh", None)) ath1 = _get_value_from_ds_attr( ds, "ath1", "dsp_ath1", value=kwargs.pop("ath1", None) @@ -157,7 +157,7 @@ def ssm(ds, gwt, sources=None, **kwargs): ssm : flopy ModflowGwtssm ssm package """ - logger.info("creating modflow SSM") + logger.info("creating mf6 SSM") build_tuples = False if sources is None: @@ -168,6 +168,10 @@ def ssm(ds, gwt, sources=None, **kwargs): if build_tuples and sources is not None: sources = [(ipkg, "AUX", "CONCENTRATION") for ipkg in sources] + if len(sources) == 0: + logger.error("No sources to add to gwt model!") + raise ValueError("No sources to add to gwt model!") + ssm = flopy.mf6.ModflowGwtssm(gwt, sources=sources, **kwargs) return ssm @@ -190,7 +194,7 @@ def mst(ds, gwt, porosity=None, **kwargs): mst : flopy ModflowGwtmst mst package """ - logger.info("creating modflow MST") + logger.info("creating mf6 MST") # NOTE: attempting to look for porosity in attributes first, then data variables. # If both are defined, the attribute value will be used. The log message in this @@ -227,7 +231,7 @@ def cnc(ds, gwt, da_mask, da_conc, pname="cnc", **kwargs): cnc : flopy ModflowGwtcnc cnc package """ - logger.info("creating modflow CNC") + logger.info("creating mf6 CNC") cnc_rec = grid.da_to_reclist(ds, da_mask, col1=da_conc, layer=None) cnc_spd = {0: cnc_rec} @@ -263,7 +267,7 @@ def oc( oc : flopy ModflowGwtoc oc package """ - logger.info("creating modflow OC") + logger.info("creating mf6 OC") # Create the output control package concfile = f"{gwt.name}.ucn" @@ -306,7 +310,7 @@ def ic(ds, gwt, strt, pname="ic", **kwargs): ic : flopy ModflowGwtic ic package """ - logger.info("creating modflow IC") + logger.info("creating mf6 IC") if not isinstance(strt, numbers.Number): strt = ds[strt].data ic = flopy.mf6.ModflowGwtic(gwt, strt=strt, pname=pname, **kwargs) @@ -315,7 +319,7 @@ def ic(ds, gwt, strt, pname="ic", **kwargs): def gwfgwt(ds, sim, exgtype="GWF6-GWT6", **kwargs): - """create GWF-GWT exchange package for modflow simulation. + """create GWF-GWT exchange package for mf6 simulation. Parameters ---------- @@ -331,7 +335,7 @@ def gwfgwt(ds, sim, exgtype="GWF6-GWT6", **kwargs): gwfgwt : _description_ """ - logger.info("creating modflow exchange GWFGWT") + logger.info("creating mf6 exchange GWFGWT") type_name_dict = {} for name, mod in sim.model_dict.items(): type_name_dict[mod.model_type] = name diff --git a/nlmod/gwt/output.py b/nlmod/gwt/output.py index 560bb0c7..87cf580d 100644 --- a/nlmod/gwt/output.py +++ b/nlmod/gwt/output.py @@ -1,36 +1,54 @@ import logging import os +import warnings import flopy import numpy as np import xarray as xr from ..dims.layers import calculate_thickness -from ..mfoutput import _get_output_da +from ..mfoutput.mfoutput import _get_heads_da, _get_time_index logger = logging.getLogger(__name__) -def _get_concentration(ds=None, gwt=None, fname_conc=None): - msg = "Load the concentration using either the ds or the gwt" - assert ((ds is not None) + (gwt is not None)) == 1, msg +def get_concentration_obj(ds=None, gwt=None, fname=None, grbfile=None): + msg = "Load the concentration using either the ds, gwt or a fname_conc" + assert ((ds is not None) + (gwt is not None) + (fname is not None)) == 1, msg - if fname_conc is None: + if fname is None: if ds is None: - concobj = gwt.output.concentration() + return gwt.output.concentration() else: - fname_conc = os.path.join(ds.model_ws, f"{ds.model_name}_gwt.ucn") - if fname_conc is not None: - concobj = flopy.utils.HeadFile(fname_conc, text="concentration") + fname = os.path.join(ds.model_ws, f"{ds.model_name}_gwt.ucn") + # get grb file + if ds.gridtype == "vertex": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".disv.grb") + elif ds.gridtype == "structured": + grbfile = os.path.join(ds.model_ws, ds.model_name + ".dis.grb") + else: + grbfile = None + if fname is not None: + if grbfile is not None: + mg = flopy.mf6.utils.MfGrdFile(grbfile).modelgrid + else: + logger.warning(msg) + warnings.warn(msg) + mg = None + concobj = flopy.utils.HeadFile(fname, text="concentration", modelgrid=mg) return concobj -def get_concentration_da(ds=None, gwt=None, fname_conc=None): - """Reads concentration file given either a dataset or a groundwater flow object. - - Note: Calling this function with ds is currently preferred over calling it - with gwt, because the layer and time coordinates can not be fully - reconstructed from gwt. +def get_concentration_da( + ds=None, + gwt=None, + fname=None, + grbfile=None, + delayed=False, + chunked=False, + **kwargs, +): + """Reads binary concentration file. Parameters ---------- @@ -38,21 +56,44 @@ def get_concentration_da(ds=None, gwt=None, fname_conc=None): Xarray dataset with model data. gwt : flopy ModflowGwt Flopy groundwater transport object. - fname_conc : path, optional + fname : path, optional Instead of loading the binary concentration file corresponding to ds or gwf load the concentration from this file. - + grbfile : str + path to file containing binary grid information, only needed if reading + output from file using fname + delayed : bool, optional + if delayed is True, do not load output data into memory, default is False. + chunked : bool, optional + chunk data array containing output, default is False. Returns ------- conc_da : xarray.DataArray concentration data array. """ - conc_da = _get_output_da( - _get_concentration, ds=ds, gwf_or_gwt=gwt, fname=fname_conc - ) - conc_da.attrs["units"] = "concentration" - return conc_da + cobj = get_concentration_obj(ds=ds, gwt=gwt, fname=fname, grbfile=grbfile) + # gwt.output.concentration() defaults to a structured grid + if gwt is not None and ds is None and fname is None: + kwargs["modelgrid"] = gwt.modelgrid + da = _get_heads_da(cobj, **kwargs) + da.attrs["units"] = "concentration" + + # set time index if ds/gwt are provided + if ds is not None or gwt is not None: + da["time"] = _get_time_index(cobj, ds=ds, gwf_or_gwt=gwt) + if ds is not None: + da["layer"] = ds.layer + + if chunked: + # chunk data array + da = da.chunk("auto") + + if not delayed: + # load into memory + da = da.compute() + + return da def get_concentration_at_gw_surface(conc, layer="layer"): @@ -94,11 +135,16 @@ def get_concentration_at_gw_surface(conc, layer="layer"): top_layer = np.take(top_layer, 0, axis=layer) coords["layer"] = (dims, conc_da.layer.data[top_layer]) ctop = xr.DataArray(ctop, dims=dims, coords=coords) + # to not confuse this coordinate with the default layer coord in nlmod + # this source_layer has dims (time, cellid) or (time, y, x) + # indicating the source layer of the concentration value for each time step + ctop = ctop.rename({"layer": "source_layer"}) return ctop def freshwater_head(ds, hp, conc, denseref=None, drhodc=None): """Calculate equivalent freshwater head from point water heads. + Heads file produced by mf6 contains point water heads. Parameters ---------- @@ -130,6 +176,8 @@ def freshwater_head(ds, hp, conc, denseref=None, drhodc=None): if "z" not in ds: if "thickness" not in ds: thickness = calculate_thickness(ds) + else: + thickness = ds.thickness z = ds["botm"] + thickness / 2.0 else: z = ds["z"] @@ -139,6 +187,7 @@ def freshwater_head(ds, hp, conc, denseref=None, drhodc=None): def pointwater_head(ds, hf, conc, denseref=None, drhodc=None): """Calculate point water head from freshwater heads. + Heads file produced by mf6 contains point water heads. Parameters ---------- diff --git a/nlmod/mfoutput.py b/nlmod/mfoutput.py deleted file mode 100644 index 38a32f22..00000000 --- a/nlmod/mfoutput.py +++ /dev/null @@ -1,131 +0,0 @@ -import logging - -import numpy as np -import xarray as xr -from flopy.utils import CellBudgetFile, HeadFile - -from .dims.resample import get_affine_mod_to_world, get_xy_mid_structured -from .dims.time import ds_time_idx - -logger = logging.getLogger(__name__) - - -def _get_output_da(reader_func, ds=None, gwf_or_gwt=None, fname=None, **kwargs): - """Reads mf6 output file given either a dataset or a gwf or gwt object. - - Note: Calling this function with ds is currently preferred over calling it - with gwf/gwt, because the layer and time coordinates can not be fully - reconstructed from gwf/gwt. - - Parameters - ---------- - ds : xarray.Dataset - xarray dataset with model data. - gwf_or_gwt : flopy ModflowGwt - flopy groundwater flow or transport object. - fname : path, optional - instead of loading the binary concentration file corresponding to ds or - gwf/gwt load the concentration from this file. - - - Returns - ------- - da : xarray.DataArray - output data array. - """ - out_obj = reader_func(ds, gwf_or_gwt, fname) - - if gwf_or_gwt is not None: - hdry = gwf_or_gwt.hdry - hnoflo = gwf_or_gwt.hnoflo - else: - hdry = -1e30 - hnoflo = 1e30 - - # check whether out_obj is BudgetFile or HeadFile based on passed kwargs - if isinstance(out_obj, CellBudgetFile): - arr = out_obj.get_data(**kwargs) - elif isinstance(out_obj, HeadFile): - arr = out_obj.get_alldata(**kwargs) - else: - raise TypeError(f"Don't know how to deal with {type(out_obj)}!") - - if isinstance(arr, list): - arr = np.stack(arr) - - arr[arr == hdry] = np.nan - arr[arr == hnoflo] = np.nan - - if gwf_or_gwt is not None: - gridtype = gwf_or_gwt.modelgrid.grid_type - else: - gridtype = ds.gridtype - - if gridtype == "vertex": - da = xr.DataArray( - data=arr[:, :, 0], - dims=("time", "layer", "icell2d"), - coords={}, - ) - - elif gridtype == "structured": - if gwf_or_gwt is not None: - try: - delr = np.unique(gwf_or_gwt.modelgrid.delr).item() - delc = np.unique(gwf_or_gwt.modelgrid.delc).item() - extent = gwf_or_gwt.modelgrid.extent - x, y = get_xy_mid_structured(extent, delr, delc) - except ValueError: # delr/delc are variable - # x, y in local coords - x, y = gwf_or_gwt.modelgrid.xycenters - else: - x = ds.x - y = ds.y - - da = xr.DataArray( - data=arr, - dims=("time", "layer", "y", "x"), - coords={ - "x": x, - "y": y, - }, - ) - else: - raise TypeError("Gridtype not supported") - - # set layer and time coordinates - if gwf_or_gwt is not None: - da.coords["layer"] = np.arange(gwf_or_gwt.modelgrid.nlay) - da.coords["time"] = ds_time_idx( - out_obj.get_times(), - start_datetime=gwf_or_gwt.modeltime.start_datetime, - time_units=gwf_or_gwt.modeltime.time_units, - ) - else: - da.coords["layer"] = ds.layer - da.coords["time"] = ds_time_idx( - out_obj.get_times(), - start_datetime=ds.time.attrs["start"], - time_units=ds.time.attrs["time_units"], - ) - - if ds is not None and "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0: - # affine = get_affine(ds) - affine = get_affine_mod_to_world(ds) - da.rio.write_transform(affine, inplace=True) - - elif gwf_or_gwt is not None and gwf_or_gwt.modelgrid.angrot != 0.0: - attrs = { - # "delr": np.unique(gwf_or_gwt.modelgrid.delr).item(), - # "delc": np.unique(gwf_or_gwt.modelgrid.delc).item(), - "xorigin": gwf_or_gwt.modelgrid.xoffset, - "yorigin": gwf_or_gwt.modelgrid.yoffset, - "angrot": gwf_or_gwt.modelgrid.angrot, - "extent": gwf_or_gwt.modelgrid.extent, - } - affine = get_affine_mod_to_world(attrs) - da.rio.write_transform(affine, inplace=True) - - da.rio.write_crs("EPSG:28992", inplace=True) - - return da diff --git a/nlmod/mfoutput/__init__.py b/nlmod/mfoutput/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nlmod/mfoutput/binaryfile.py b/nlmod/mfoutput/binaryfile.py new file mode 100644 index 00000000..b10ae78a --- /dev/null +++ b/nlmod/mfoutput/binaryfile.py @@ -0,0 +1,220 @@ +import numpy as np +from flopy.utils.binaryfile import binaryread + + +def _get_binary_head_data(kstpkper, fobj): + """Get head data array for timestep (kstp, kper). + + Adapted from flopy.utils.binaryfile.HeadFile. Removed support for all other + types of indexing (ids/totim) and only supports kstpkper. + + Parameters + ---------- + kstpkper : tuple(int, int) + step and period index + filepath : str + path to binary heads file + hobj : flopy.utils.HeadFile + flopy HeadFile object + + Returns + ------- + np.array + array containing data for timestep (kstp, kper) + """ + kstp1 = kstpkper[0] + 1 + kper1 = kstpkper[1] + 1 + idx = np.where( + (fobj.recordarray["kstp"] == kstp1) & (fobj.recordarray["kper"] == kper1) + ) + totim1 = fobj.recordarray[idx]["totim"][0] + + keyindices = np.where(fobj.recordarray["totim"] == totim1)[0] + # initialize head with nan and then fill it + idx = keyindices[0] + nrow = fobj.recordarray["nrow"][idx] + ncol = fobj.recordarray["ncol"][idx] + arr = np.empty((fobj.nlay, nrow, ncol), dtype=fobj.realtype) + arr[:, :, :] = np.nan + for idx in keyindices: + ipos = fobj.iposarray[idx] + ilay = fobj.recordarray["ilay"][idx] + with open(fobj.filename, "rb") as f: + f.seek(ipos, 0) + nrow = fobj.recordarray["nrow"][idx] + ncol = fobj.recordarray["ncol"][idx] + shp = (nrow, ncol) + arr[ilay - 1] = binaryread(f, fobj.realtype, shp) + return arr + + +def __create3D(data, fobj): + """Copy of CellBudgetFile.__create3D. + + See flopy.utils.binaryfile.CellBudgetFile.__create3D. + """ + out = np.ma.zeros(fobj.nnodes, dtype=np.float32) + out.mask = True + for [node, q] in zip(data["node"], data["q"]): + idx = node - 1 + out.data[idx] += q + out.mask[idx] = False + return np.ma.reshape(out, fobj.shape) + + +def _select_data_indices_budget(fobj, text, kstpkper): + """Select data indices for budgetfile. + + Parameters + ---------- + fobj : flopy.utils.CellBudgetFile + CellBudgetFile object + text : str + text indicating which dataset to load + kstpkper : tuple(int, int) + step and period index + + Returns + ------- + select_indices : np.array of int + array with indices of data to load + """ + # check and make sure that text is in file + text16 = None + if text is not None: + text16 = fobj._find_text(text) + + # get kstpkper indices + kstp1 = kstpkper[0] + 1 + kper1 = kstpkper[1] + 1 + if text is None: + select_indices = np.where( + (fobj.recordarray["kstp"] == kstp1) & (fobj.recordarray["kper"] == kper1) + ) + else: + if text is not None: + select_indices = np.where( + (fobj.recordarray["kstp"] == kstp1) + & (fobj.recordarray["kper"] == kper1) + & (fobj.recordarray["text"] == text16) + ) + + # build and return the record list + if isinstance(select_indices, tuple): + select_indices = select_indices[0] + return select_indices + + +def _get_binary_budget_data(kstpkper, fobj, text): + """Get budget data from binary CellBudgetFile. + + Code copied from flopy.utils.binaryfile.CellBudgetFile and adapted to + open binary file for each function call instead of relying on one open file. + All support for totim/idx selection is dropped, only providing kstpkper will + work. + + Parameters + ---------- + kstpkper : tuple(int, int) + tuple with timestep and stressperiod indices + fobj : flopy.utils.HeadFile or flopy.utils.CellBudgetFile + file object + text : str + text indicating which dataset to read + + Returns + ------- + data : np.array + array containing data for a specific timestep + """ + # select indices to read + idx = _select_data_indices_budget(fobj, text, kstpkper) + + # idx must be an ndarray, so if it comes in as an integer then convert + if np.isscalar(idx): + idx = np.array([idx]) + + header = fobj.recordarray[idx] + ipos = fobj.iposarray[idx].item() + imeth = header["imeth"][0] + + t = header["text"][0] + if isinstance(t, bytes): + t = t.decode("utf-8") + + nlay = abs(header["nlay"][0]) + nrow = header["nrow"][0] + ncol = header["ncol"][0] + + # default method + with open(fobj.filename, "rb") as f: + f.seek(ipos, 0) + + # imeth 0 + if imeth == 0: + return binaryread(f, fobj.realtype(1), shape=(nlay, nrow, ncol)) + # imeth 1 + elif imeth == 1: + return binaryread(f, fobj.realtype(1), shape=(nlay, nrow, ncol)) + + # imeth 2 + elif imeth == 2: + nlist = binaryread(f, np.int32)[0] + dtype = np.dtype([("node", np.int32), ("q", fobj.realtype)]) + data = binaryread(f, dtype, shape=(nlist,)) + return __create3D(data, fobj) + + # imeth 3 + elif imeth == 3: + ilayer = binaryread(f, np.int32, shape=(nrow, ncol)) + data = binaryread(f, fobj.realtype(1), shape=(nrow, ncol)) + out = np.ma.zeros(fobj.nnodes, dtype=np.float32) + out.mask = True + vertical_layer = ilayer.flatten() - 1 + # create the 2D cell index and then move it to + # the correct vertical location + idx = np.arange(0, vertical_layer.shape[0]) + idx += vertical_layer * nrow * ncol + out[idx] = data.flatten() + return out.reshape(fobj.shape) + + # imeth 4 + elif imeth == 4: + return binaryread(f, fobj.realtype(1), shape=(nrow, ncol)) + + # imeth 5 + elif imeth == 5: + nauxp1 = binaryread(f, np.int32)[0] + naux = nauxp1 - 1 + dtype_list = [("node", np.int32), ("q", fobj.realtype)] + for _ in range(naux): + auxname = binaryread(f, str, charlen=16) + if not isinstance(auxname, str): + auxname = auxname.decode() + dtype_list.append((auxname.strip(), fobj.realtype)) + dtype = np.dtype(dtype_list) + nlist = binaryread(f, np.int32)[0] + data = binaryread(f, dtype, shape=(nlist,)) + return __create3D(data, fobj) + + # imeth 6 + elif imeth == 6: + # read rest of list data + nauxp1 = binaryread(f, np.int32)[0] + naux = nauxp1 - 1 + dtype_list = [("node", np.int32), ("node2", np.int32), ("q", fobj.realtype)] + for _ in range(naux): + auxname = binaryread(f, str, charlen=16) + if not isinstance(auxname, str): + auxname = auxname.decode() + dtype_list.append((auxname.strip(), fobj.realtype)) + dtype = np.dtype(dtype_list) + nlist = binaryread(f, np.int32)[0] + data = binaryread(f, dtype, shape=(nlist,)) + data = __create3D(data, fobj) + if fobj.modelgrid is not None: + return np.reshape(data, fobj.shape) + else: + return data + else: + raise ValueError(f"invalid imeth value - {imeth}") diff --git a/nlmod/mfoutput/mfoutput.py b/nlmod/mfoutput/mfoutput.py new file mode 100644 index 00000000..69bc5164 --- /dev/null +++ b/nlmod/mfoutput/mfoutput.py @@ -0,0 +1,219 @@ +import logging + +import dask +import xarray as xr + +from ..dims.grid import get_dims_coords_from_modelgrid +from ..dims.resample import get_affine_mod_to_world +from ..dims.time import ds_time_idx +from .binaryfile import _get_binary_budget_data, _get_binary_head_data + +logger = logging.getLogger(__name__) + + +def _get_dask_array(func, kstpkper, shape, **kwargs): + """Get stacked dask array for given timesteps. + + Parameters + ---------- + func : function + function that returns a numpy array + kstpkper : list of tuples + list of tuples containing (timestep, stressperiod) indices. + shape : tuple + shape of array that is returned by func + **kwargs + additional kwargs passed to func + + Returns + ------- + dask.array + stacked dask array + """ + result = [] + for ki in kstpkper: + d = dask.delayed(func)(kstpkper=ki, **kwargs) + arr = dask.array.from_delayed(d, shape=shape, dtype=float) + result.append(arr) + return dask.array.stack(result) + + +def _get_time_index(fobj, ds=None, gwf_or_gwt=None): + """Get time index based on flopy binaryfile object. + + Binary files objects are e.g. flopy.utils.HeadFile, flopy.utils.CellBudgetFile. + + Parameters + ---------- + fobj : flopy.utils.HeadFile or flopy.utils.CellBudgetFile + flopy binary file object + ds : xarray.Dataset, optional + model dataset, by default None + gwf_or_gwt : flopy.mf6.ModflowGwf or flopy.mf6.ModflowGwt, optional + flopy groundwater flow or transport model, by default None + + Returns + ------- + tindex : xarray.IndexVariable + index variable with time converted to timestamps + """ + # set layer and time coordinates + if gwf_or_gwt is not None: + tindex = ds_time_idx( + fobj.get_times(), + start_datetime=gwf_or_gwt.modeltime.start_datetime, + time_units=gwf_or_gwt.modeltime.time_units, + ) + elif ds is not None: + tindex = ds_time_idx( + fobj.get_times(), + start_datetime=ds.time.attrs["start"], + time_units=ds.time.attrs["time_units"], + ) + return tindex + + +def _create_da(arr, modelgrid, times): + """Create data array based on array, modelgrid, and time array. + + Parameters + ---------- + arr : dask.array or numpy.array + array containing data + modelgrid : flopy.discretization.Grid + flopy modelgrid object + times : list or array + list or array containing times as floats (usually in days) + + Returns + ------- + da : xarray.DataArray + data array with spatial dimensions based on modelgrid and + time dimension based on times + """ + # create data array + dims, coords = get_dims_coords_from_modelgrid(modelgrid) + da = xr.DataArray(data=arr, dims=("time",) + dims, coords=coords) + + # set dry/no-flow to nan + hdry = -1e30 + hnoflo = 1e30 + da = da.where((da != hdry) & (da != hnoflo)) + + # set local time coordinates + da.coords["time"] = ds_time_idx(times) + + # set affine if angrot != 0.0 + if modelgrid.angrot != 0.0: + attrs = { + "xorigin": modelgrid.xoffset, + "yorigin": modelgrid.yoffset, + "angrot": modelgrid.angrot, + "extent": modelgrid.extent, + } + affine = get_affine_mod_to_world(attrs) + da.rio.write_transform(affine, inplace=True) + + # write CRS + da.rio.write_crs("EPSG:28992", inplace=True) + return da + + +def _get_heads_da( + hobj, + modelgrid=None, + **kwargs, +): + """Get heads data array based on HeadFile object. + + Optionally provide modelgrid separately if HeadFile object does not contain + correct grid definition. + + Parameters + ---------- + hobj : flopy.utils.HeadFile + flopy HeadFile object for binary heads + modelgrid : flopy.discretization.Grid, optional + flopy modelgrid object, default is None, in which case the modelgrid + is derived from `hobj.mg` + + Returns + ------- + da : xarray.DataArray + output data array + """ + if "kstpkper" in kwargs: + kstpkper = kwargs.pop("kstpkper") + else: + kstpkper = hobj.get_kstpkper() + + if modelgrid is None: + modelgrid = hobj.mg + # shape is derived from hobj, not modelgrid as array read from + # binary file always has 3 dimensions + shape = (hobj.nlay, hobj.nrow, hobj.ncol) + + # load data from binary file + stacked_arr = _get_dask_array( + _get_binary_head_data, kstpkper=kstpkper, shape=shape, fobj=hobj + ) + + # check for vertex grids + if modelgrid.grid_type == "vertex": + if stacked_arr.ndim == 4: + stacked_arr = stacked_arr[:, :, 0, :] + + # create data array + da = _create_da(stacked_arr, modelgrid, hobj.get_times()) + + return da + + +def _get_budget_da( + cbcobj, + text, + modelgrid=None, + **kwargs, +): + """Get budget data array based on CellBudgetFile and text string. + + Optionally provide modelgrid separately if CellBudgetFile object does not contain + correct grid definition. + + Parameters + ---------- + cbcobj : flopy.utils.CellBudgetFile + flopy HeadFile object for binary heads + text: str + string indicating which dataset to load from budget file + modelgrid : flopy.discretization.Grid, optional + flopy modelgrid object, default is None, in which case the modelgrid + is derived from `cbcobj.modelgrid` + + Returns + ------- + da : xarray.DataArray + output data array. + """ + if "kstpkper" in kwargs: + kstpkper = kwargs.pop("kstpkper") + else: + kstpkper = cbcobj.get_kstpkper() + + if modelgrid is None: + modelgrid = cbcobj.modelgrid + + # load data from binary file + shape = modelgrid.shape + stacked_arr = _get_dask_array( + _get_binary_budget_data, + kstpkper=kstpkper, + shape=shape, + fobj=cbcobj, + text=text, + ) + + # create data array + da = _create_da(stacked_arr, modelgrid, cbcobj.get_times()) + + return da diff --git a/nlmod/modpath/modpath.py b/nlmod/modpath/modpath.py index 51b51081..20db899e 100644 --- a/nlmod/modpath/modpath.py +++ b/nlmod/modpath/modpath.py @@ -15,39 +15,34 @@ logger = logging.getLogger(__name__) -def write_and_run(mpf, remove_prev_output=True, nb_path=None): - """write modpath files and run the model. - - 2 extra options: - 1. remove output of the previous run - 2. copy the modelscript (typically a Jupyter Notebook) to the model - workspace with a timestamp. - +def write_and_run(mpf, remove_prev_output=True, script_path=None, silent=False): + """write modpath files and run the model. Extra options include removing + previous output and copying the modelscript to the model workspace. Parameters ---------- mpf : flopy.modpath.mp7.Modpath7 modpath model. - model_ds : xarray.Dataset - dataset with model data. remove_prev_output : bool, optional remove the output of a previous modpath run (if it exists) - nb_path : str or None, optional - full path of the Jupyter Notebook (.ipynb) with the modelscript. The + script_path : str or None, optional + full path of the Jupyter Notebook (.ipynb) or a script (.py). The default is None. Preferably this path does not have to be given manually but there is currently no good option to obtain the filename of a Jupyter Notebook from within the notebook itself. + silent : bool, optional + run model silently """ if remove_prev_output: remove_output(mpf) - if nb_path is not None: - new_nb_fname = ( - f'{dt.datetime.now().strftime("%Y%m%d")}' + os.path.split(nb_path)[-1] + if script_path is not None: + new_fname = ( + f'{dt.datetime.now().strftime("%Y%m%d")}' + os.path.split(script_path)[-1] ) - dst = os.path.join(mpf.model_ws, new_nb_fname) - logger.info(f"write script {new_nb_fname} to modpath workspace") - copyfile(nb_path, dst) + dst = os.path.join(mpf.model_ws, new_fname) + logger.info(f"write script {new_fname} to modpath workspace") + copyfile(script_path, dst) logger.info("write modpath files to model workspace") @@ -56,7 +51,7 @@ def write_and_run(mpf, remove_prev_output=True, nb_path=None): # run modpath logger.info("run modpath model") - assert mpf.run_model()[0], "Modpath run not succeeded" + assert mpf.run_model(silent=silent)[0], "Modpath run not succeeded" def xy_to_nodes(xy_list, mpf, ds, layer=0): @@ -118,10 +113,14 @@ def package_to_nodes(gwf, package_name, mpf): node numbers corresponding to the cells with a certain boundary condition. """ gwf_package = gwf.get_package(package_name) - if not hasattr(gwf_package, "stress_period_data"): - raise TypeError("only package with stress period data can be used") - - pkg_cid = gwf_package.stress_period_data.array[0]["cellid"] + if hasattr(gwf_package, "stress_period_data"): + pkg_cid = gwf_package.stress_period_data.array[0]["cellid"] + elif hasattr(gwf_package, "connectiondata"): + pkg_cid = gwf_package.connectiondata.array["cellid"] + else: + raise TypeError( + "only package with stress period data or connectiondata can be used" + ) nodes = [] for cid in pkg_cid: if mpf.ib[cid[0], cid[1]] > 0: @@ -201,12 +200,6 @@ def mpf(gwf, exe_name=None, modelname=None, model_ws=None): "the save_flows option of the npf package should be True not None" ) - # check if the tdis has a start_time - if gwf.simulation.tdis.start_date_time.array is not None: - logger.warning( - "older versions of modpath cannot handle this, see https://github.com/MODFLOW-USGS/modpath-v7/issues/31" - ) - # get executable if exe_name is None: exe_name = util.get_exe_path("mp7_2_002_provisional") @@ -436,15 +429,25 @@ def pg_from_pd(nodes, localx=0.5, localy=0.5, localz=0.5): return pg -def sim(mpf, pg, direction="backward", gwf=None, ref_time=None, stoptime=None): +def sim( + mpf, + particlegroups, + direction="backward", + gwf=None, + ref_time=None, + stoptime=None, + simulationtype="combined", + weaksinkoption="pass_through", + weaksourceoption="pass_through", +): """Create a modpath backward simulation from a particle group. Parameters ---------- mpf : flopy.modpath.mp7.Modpath7 modpath object. - pg : flopy.modpath.mp7particlegroup.ParticleGroupNodeTemplate - Particle group. + particlegroups : ParticleGroup or list of ParticleGroups + One or more particle groups. gwf : flopy.mf6.mfmodel.MFModel or None, optional Groundwater flow model. Only used if ref_time is not None. Default is None @@ -477,14 +480,14 @@ def sim(mpf, pg, direction="backward", gwf=None, ref_time=None, stoptime=None): mpsim = flopy.modpath.Modpath7Sim( mpf, - simulationtype="combined", + simulationtype=simulationtype, trackingdirection=direction, - weaksinkoption="pass_through", - weaksourceoption="pass_through", + weaksinkoption=weaksinkoption, + weaksourceoption=weaksourceoption, referencetime=ref_time, stoptimeoption=stoptimeoption, stoptime=stoptime, - particlegroups=pg, + particlegroups=particlegroups, ) return mpsim diff --git a/nlmod/plot/dcs.py b/nlmod/plot/dcs.py index 2a802c84..52fb7fde 100644 --- a/nlmod/plot/dcs.py +++ b/nlmod/plot/dcs.py @@ -6,9 +6,11 @@ import xarray as xr from matplotlib.collections import LineCollection, PatchCollection from matplotlib.patches import Rectangle +from shapely.affinity import affine_transform from shapely.geometry import LineString, MultiLineString, Point, Polygon from ..dims.grid import modelgrid_from_ds +from ..dims.resample import get_affine_world_to_mod class DatasetCrossSection: @@ -48,6 +50,8 @@ def __init__( self.icell2d = icell2d if "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0: + # transform the line to model coordinates + line = affine_transform(line, get_affine_world_to_mod(ds).to_shapely()) self.rotated = True else: self.rotated = False @@ -55,7 +59,7 @@ def __init__( # first determine where the cross-section crosses grid-lines if self.icell2d in ds.dims: # determine the cells that are crossed - modelgrid = modelgrid_from_ds(ds) + modelgrid = modelgrid_from_ds(ds, rotated=False) gi = flopy.utils.GridIntersect(modelgrid, method="vertex") r = gi.intersect(line) s_cell = [] @@ -64,7 +68,7 @@ def __init__( if intersection.length == 0: continue if isinstance(intersection, MultiLineString): - for ix in intersection: + for ix in intersection.geoms: s_cell.append([line.project(Point(ix.coords[0])), 1, ic2d]) s_cell.append([line.project(Point(ix.coords[-1])), 0, ic2d]) continue @@ -156,7 +160,14 @@ def line_intersect_grid(self, cs_line): xys = xys[xys[:, -1].argsort()] return xys - def plot_layers(self, colors=None, min_label_area=np.inf, **kwargs): + def plot_layers( + self, + colors=None, + min_label_area=np.inf, + fontsize=None, + only_labels=False, + **kwargs, + ): if colors is None: cmap = plt.get_cmap("tab20") colors = [cmap(i) for i in range(len(self.layer))] @@ -198,8 +209,9 @@ def plot_layers(self, colors=None, min_label_area=np.inf, **kwargs): # xy = np.vstack((x, y)).T color = colors[i] pol = matplotlib.patches.Polygon(xy, facecolor=color, **kwargs) - self.ax.add_patch(pol) - polygons.append(pol) + if not only_labels: + self.ax.add_patch(pol) + polygons.append(pol) if not np.isinf(min_label_area): pols = Polygon(xy) @@ -217,15 +229,25 @@ def plot_layers(self, colors=None, min_label_area=np.inf, **kwargs): yp = list(reversed(y[int(len(x) / 2) :])) yp2 = np.interp(xt, xp, yp) yt = np.mean([yp1, yp2]) - self.ax.text( + ht = self.ax.text( xt, yt, self.layer[i], ha="center", va="center", + fontsize=fontsize, ) + if only_labels: + polygons.append(ht) return polygons + def label_layers(self, min_label_area=None): + if min_label_area is None: + # plot labels of layers with an average thickness of 1 meter + # in entire cross-section + min_label_area = self.line.length * 1 + return self.plot_layers(min_label_area=min_label_area, only_labels=True) + def plot_grid( self, edgecolor="k", @@ -304,6 +326,8 @@ def plot_array(self, z, head=None, **kwargs): if isinstance(z, xr.DataArray): z = z.data if head is not None: + if isinstance(head, xr.DataArray): + head = head.data assert head.shape == z.shape if self.icell2d in self.ds.dims: assert len(z.shape) == 2 diff --git a/nlmod/plot/flopy.py b/nlmod/plot/flopy.py index 58ced313..b6be4b43 100644 --- a/nlmod/plot/flopy.py +++ b/nlmod/plot/flopy.py @@ -1,4 +1,3 @@ -import os from functools import partial import flopy @@ -49,12 +48,68 @@ def map_array( alpha=1.0, colorbar=True, colorbar_label="", - plot_grid=True, + plot_grid=False, add_to_plot=None, - backgroundmap=False, + background=False, figsize=None, animate=False, ): + """Plot an array using flopy PlotMapView. + + Parameters + ---------- + arr : np.array, xarray.DataArray + array to plot + gwf : flopy.mf6.ModflowGwf or flopy.mf6.ModflowGwt + flopy groundwater flow or transport model + ilay : int, optional + layer to plot, by default 0 + iper : int, optional + timestep to plot, by default 0 + extent : list or tuple, optional + plot extent: (xmin, xmax, ymin, ymax), by default None which defaults + model extent + ax : matplotlib Axes, optional + axis handle to plot on, by default None + title : str, optional + title of the plot, by default "" (blank) + xlabel : str, optional + x-axis label, by default "X [km RD]" + ylabel : str, optional + y-axis label, by default "Y [km RD]" + norm : matplotlib.colors.Norm + colorbar norm + vmin : float, optional + minimum value for colorbar + vmax : float, optional + maximum value for colorbar + levels : np.array, optional + colorbar levels, used for setting colorbar ticks + cmap : str or colormap, optional + colormap, default is "viridis" + alpha : float, optional + transparency, by default 1.0 + plot_grid : bool, optional + plot model grid, by default False + add_to_plot : tuple of func, optional + tuple or list of plotting functions that take ax as the + only argument, by default None. Use to add features to plot, e.g. + plotting shapefiles, or other data. + background : bool, optional + add background map, by default False + figsize : tuple, optional + figure size, by default None + animate : bool, optional + if True return figure, axis and quadmesh handles, by default + False (returns only axes handle) + + Returns + ------- + ax : matplotlib Axes + axes handle + f, ax, qm : + only if animate is True, return figure, axes and quadmesh handles. + """ # get data if isinstance(arr, xr.DataArray): arr = arr.values @@ -79,7 +134,7 @@ def map_array( qm = pmv.plot_array(arr, cmap=cmap, norm=norm, alpha=alpha) # bgmap - if backgroundmap: + if background: add_background_map(ax, map_provider="nlmaps.water", alpha=0.5) # add other info to plot @@ -111,6 +166,127 @@ def map_array( return ax +def contour_array( + arr, + gwf, + ilay=0, + iper=0, + extent=None, + ax=None, + title="", + xlabel="X [km RD]", + ylabel="Y [km RD]", + levels=10, + alpha=1.0, + labels=True, + label_kwargs=None, + plot_grid=False, + add_to_plot=None, + background=False, + figsize=None, + animate=False, + **kwargs, +): + """Contour an array using flopy PlotMapView. + + Parameters + ---------- + arr : np.array, xarray.DataArray + array to contour + gwf : flopy.mf6.ModflowGwf or flopy.mf6.ModflowGwt + flopy groundwater flow or transport model + ilay : int, optional + layer to contour, by default 0 + iper : int, optional + timestep to contour, by default 0 + extent : list or tuple, optional + plot extent: (xmin, xmax, ymin, ymax), by default None which defaults + model extent + ax : matplotlib Axes, optional + axis handle to plot on, by default None + title : str, optional + title of the plot, by default "" (blank) + xlabel : str, optional + x-axis label, by default "X [km RD]" + ylabel : str, optional + y-axis label, by default "Y [km RD]" + levels : int, list, np.array, optional + contour levels, when passed as int draw that many contours, when + list or array draw contours at provided levels, by default 10 + alpha : float, optional + transparency of contour lines, by default 1.0 + labels : bool, optional + add contour labels showing contour levels, by default True + label_kwargs : dict, optional + keyword arguments passed onto ax.clabel(), by default None + plot_grid : bool, optional + plot model grid, by default False + add_to_plot : tuple of func, optional + tuple or list of plotting functions that take ax as the + only argument, by default None. Use to add features to plot, e.g. + plotting shapefiles, or other data. + background : bool, optional + add background map, by default False + figsize : tuple, optional + figure size, by default None + animate : bool, optional + if True return figure, axis and contour handles, by default + False (returns only axes handle) + + Returns + ------- + ax : matplotlib Axes + axes handle + f, ax, cs : + only if animate is True, return figure, axes and contour handles. + """ + # get data + if isinstance(arr, xr.DataArray): + arr = arr.values + + # get correct timestep and layer if need be + if len(arr.shape) == 4: + arr = arr[iper] + if len(arr.shape) == 3: + arr = arr[ilay] + + # get figure + f, ax = _get_figure(ax=ax, gwf=gwf, figsize=figsize) + + # get plot obj + pmv = flopy.plot.PlotMapView(gwf, layer=ilay, ax=ax, extent=extent) + + # plot data + cs = pmv.contour_array(arr, levels=levels, alpha=alpha, **kwargs) + if labels: + if label_kwargs is None: + label_kwargs = {} + ax.clabel(cs, **label_kwargs) + + # bgmap + if background: + add_background_map(ax, map_provider="nlmaps.water", alpha=0.5) + + # add other info to plot + if add_to_plot is not None: + for fplot in add_to_plot: + fplot(ax) + + if plot_grid: + pmv.plot_grid(lw=0.25, alpha=0.5) + + # axes properties + axprops = {"xlabel": xlabel, "ylabel": ylabel, "title": title} + ax.set(**axprops) + + f.tight_layout() + + if animate: + return f, ax, cs + else: + return ax + + def animate_map( arr, times, @@ -132,7 +308,7 @@ def animate_map( colorbar_label="", plot_grid=True, add_to_plot=None, - backgroundmap=False, + background=False, figsize=(9.24, 10.042), save=False, fname=None, @@ -170,7 +346,7 @@ def animate_map( colorbar_label=colorbar_label, plot_grid=plot_grid, add_to_plot=add_to_plot, - backgroundmap=backgroundmap, + background=background, figsize=figsize, animate=True, ) diff --git a/nlmod/plot/plot.py b/nlmod/plot/plot.py index 867404a4..f581ff78 100644 --- a/nlmod/plot/plot.py +++ b/nlmod/plot/plot.py @@ -176,9 +176,15 @@ def data_array(da, ds=None, ax=None, rotated=False, edgecolor=None, **kwargs): """ if ax is None: ax = plt.gca() + if "layer" in da.dims: + msg = ( + "The suppplied DataArray in nlmod.plot.data_darray contains multiple " + "layers. Please select a layer first." + ) + raise (ValueError(msg)) if "icell2d" in da.dims: if ds is None: - raise (Exception("Supply model dataset (ds) for grid information")) + raise (ValueError("Supply model dataset (ds) for grid information")) if isinstance(ds, list): patches = ds else: @@ -211,11 +217,11 @@ def geotop_lithok_in_cross_section( Parameters ---------- - line : sahpely.LineString + line : shapely.LineString The line along which the GeoTOP data is plotted gt : xr.Dataset, optional The voxel-dataset from GeoTOP. It is downloaded with the method - nlmod.read.geaotop.get_geotop_raw_within_extent if None. The default is None. + `nlmod.read.geotop.get_geotop()` if None. The default is None. ax : matplotlib.Axes, optional The axes in whcih the cross-section is plotted. Will default to the current axes if None. The default is None. @@ -244,7 +250,7 @@ def geotop_lithok_in_cross_section( x = [coord[0] for coord in line.coords] y = [coord[1] for coord in line.coords] extent = [min(x), max(x), min(y), max(y)] - gt = geotop.get_geotop_raw_within_extent(extent) + gt = geotop.get_geotop(extent) if "top" not in gt or "botm" not in gt: gt = geotop.add_top_and_botm(gt) @@ -253,7 +259,7 @@ def geotop_lithok_in_cross_section( lithok_props = geotop.get_lithok_props() cs = DatasetCrossSection(gt, line, layer="z", ax=ax, **kwargs) - lithoks = gt["lithok"].data + lithoks = gt["lithok"].values lithok_un = np.unique(lithoks[~np.isnan(lithoks)]) array = np.full(lithoks.shape, np.NaN) @@ -296,8 +302,12 @@ def _get_figure(ax=None, da=None, ds=None, figsize=None, rotated=True): # try to ensure pixel size is divisible by 2 figsize = (figsize[0], np.round(figsize[1] / 0.02, 0) * 0.02) - base = 10 ** int(np.log10(extent[1] - extent[0])) - f, ax = get_map(extent, base=base, figsize=figsize, tight_layout=False) + base = 10 ** int(np.log10(extent[1] - extent[0])) / 2 + if base < 1000: + fmt = "{:.1f}" + else: + fmt = "{:.0f}" + f, ax = get_map(extent, base=base, figsize=figsize, tight_layout=False, fmt=fmt) ax.set_aspect("equal", adjustable="box") return f, ax @@ -324,7 +334,7 @@ def map_array( plot_grid=True, rotated=True, add_to_plot=None, - backgroundmap=False, + background=False, figsize=None, animate=False, ): @@ -336,10 +346,10 @@ def map_array( try: nlay = da["layer"].shape[0] except IndexError: - nlay = 1 # only one layer + nlay = 0 # only one layer except KeyError: nlay = -1 # no dim layer - if nlay > 1: + if nlay >= 1: layer = da["layer"].isel(layer=ilay).item() da = da.isel(layer=ilay) elif nlay < 0: @@ -353,10 +363,14 @@ def map_array( try: nper = da["time"].shape[0] except IndexError: - nper = 1 - if nper > 1: + nper = 0 # only one timestep + except KeyError: + nper = -1 # no dim time + if nper >= 1: t = pd.Timestamp(da["time"].isel(time=iper).item()) da = da.isel(time=iper) + elif nper < 0: + iper = None else: iper = 0 t = pd.Timestamp(ds["time"].item()) @@ -379,7 +393,7 @@ def map_array( ax.axis(extent) # bgmap - if backgroundmap: + if background: add_background_map(ax, map_provider="nlmaps.water", alpha=0.5) # add other info to plot @@ -436,7 +450,7 @@ def animate_map( colorbar_label="", plot_grid=True, rotated=True, - backgroundmap=False, + background=False, figsize=None, ax=None, add_to_plot=None, @@ -483,7 +497,7 @@ def animate_map( Whether to plot the model grid. Default is True. rotated : bool, optional Whether to plot rotated model, if applicable. Default is True. - backgroundmap : bool, optional + background : bool, optional Whether to add a background map. Default is False. figsize : tuple, optional figure size in inches, default is None. @@ -514,7 +528,10 @@ def animate_map( """ # if da is a string and ds is provided select data array from model dataset if isinstance(da, str) and ds is not None: - da = ds[da] + da = ds[da].isel(layer=ilay) + else: + if "layer" in da.dims: + da = da.isel(layer=ilay) # check da if "time" not in da.dims: @@ -542,13 +559,13 @@ def animate_map( plot_grid=plot_grid, rotated=rotated, add_to_plot=add_to_plot, - backgroundmap=backgroundmap, + background=background, figsize=figsize, animate=True, ) # remove timestamp from title axtitle = ax.get_title() - axtitle.set_title(axtitle.replace("(t=", "(tstart=")) + ax.set_title(axtitle.replace("(t=", "(tstart=")) # add updating title t = pd.Timestamp(da.time.values[0]) @@ -585,6 +602,8 @@ def update(iper, pc, title): # save animation as mp4 if save: + if fname is None: + raise ValueError("please specify a fname or use save=False") writer = FFMpegWriter( fps=10, bitrate=-1, diff --git a/nlmod/plot/plotutil.py b/nlmod/plot/plotutil.py index dffb6970..00c134e3 100644 --- a/nlmod/plot/plotutil.py +++ b/nlmod/plot/plotutil.py @@ -4,6 +4,7 @@ from matplotlib.ticker import FuncFormatter, MultipleLocator from ..dims.resample import get_affine_mod_to_world +from ..epsg28992 import EPSG_28992 def get_patches(ds, rotated=False): @@ -14,8 +15,8 @@ def get_patches(ds, rotated=False): affine = get_affine_mod_to_world(ds) xy[:, 0], xy[:, 1] = affine * (xy[:, 0], xy[:, 1]) icvert = ds["icvert"].data - if "_FillValue" in ds["icvert"].attrs: - nodata = ds["icvert"].attrs["_FillValue"] + if "nodata" in ds["icvert"].attrs: + nodata = ds["icvert"].attrs["nodata"] else: nodata = -1 icvert = icvert.copy() @@ -53,7 +54,7 @@ def get_providers(provider): return providers -def add_background_map(ax, crs=28992, map_provider="nlmaps.standaard", **kwargs): +def add_background_map(ax, crs=EPSG_28992, map_provider="nlmaps.standaard", **kwargs): """Add background map to axes using contextily. Parameters @@ -84,10 +85,11 @@ def get_map( nrows=1, ncols=1, base=1000.0, + fmt_base=1000.0, fmt="{:.0f}", sharex=False, sharey=True, - crs=28992, + crs=EPSG_28992, background=False, alpha=0.5, tight_layout=True, @@ -106,8 +108,9 @@ def get_map( ncols : int, optional The number of columns. The default is 1. base : float, optional - The interval for ticklabels on the x- and y-axis. The default is 1000. - m. + The interval for ticklabels on the x- and y-axis. The default is 1000 m. + fmt_base : float, optional + divide ticklabels by this number, by default 1000, so units become km. fmt : string, optional The format of the ticks on the x- and y-axis. The default is "{:.0f}". sharex : bool, optional @@ -117,10 +120,10 @@ def get_map( Only display the ticks on the left y-axes, when ncols > 1. The default is True. background : bool or str, optional - Draw a background using contextily when True or when background is a string. + Draw a background map using contextily when True or when background is a string. When background is a string it repesents the map-provider. Use nlmod.plot._list_contextily_providers().keys() to show possible map-providers. - THe defaults is False. + The defaults is False. alpha: float, optional The alpha value of the background. The default is 0.5. tight_layout : bool, optional @@ -152,7 +155,7 @@ def set_ax_in_map(ax): ax.set_xticks([]) ax.set_yticks([]) else: - rd_ticks(ax, base=base, fmt=fmt) + rd_ticks(ax, base=base, fmt=fmt, fmt_base=fmt_base) if background: add_background_map(ax, crs=crs, map_provider=background, alpha=alpha) @@ -214,7 +217,7 @@ def colorbar_inside( cax.yaxis.tick_left() cax.yaxis.set_label_position("left") if isinstance(bbox_labels, bool) and bbox_labels is True: - bbox_labels = dict(facecolor="w", alpha=0.5) + bbox_labels = {"facecolor": "w", "alpha": 0.5} if isinstance(bbox_labels, dict): for label in cb.ax.yaxis.get_ticklabels(): label.set_bbox(bbox_labels) @@ -237,7 +240,7 @@ def title_inside( ax = plt.gca() if isinstance(bbox, bool): if bbox: - bbox = dict(facecolor="w", alpha=0.5) + bbox = {"facecolor": "w", "alpha": 0.5} else: bbox = None return ax.text( diff --git a/nlmod/read/__init__.py b/nlmod/read/__init__.py index f99c972e..efbcfac4 100644 --- a/nlmod/read/__init__.py +++ b/nlmod/read/__init__.py @@ -1,6 +1,8 @@ from . import ( ahn, bgt, + boundaries, + bro, brp, geotop, jarkus, @@ -11,5 +13,7 @@ rws, waterboard, webservices, + nhi, ) +from .geotop import get_geotop from .regis import get_regis diff --git a/nlmod/read/ahn.py b/nlmod/read/ahn.py index 97665d8d..8f0ce8cf 100644 --- a/nlmod/read/ahn.py +++ b/nlmod/read/ahn.py @@ -54,7 +54,7 @@ def get_ahn(ds=None, identifier="AHN4_DTM_5m", method="average", extent=None): elif version == 4: ahn_ds_raw = get_ahn4(extent, identifier=identifier) else: - raise (Exception(f"Unknown ahn-version: {version}")) + raise (ValueError(f"Unknown ahn-version: {version}")) ahn_ds_raw = ahn_ds_raw.drop_vars("band") @@ -69,7 +69,7 @@ def get_ahn(ds=None, identifier="AHN4_DTM_5m", method="average", extent=None): if ds is None: return ahn_da - ds_out = get_ds_empty(ds) + ds_out = get_ds_empty(ds, keep_coords=("y", "x")) ds_out["ahn"] = ahn_da return ds_out @@ -98,7 +98,7 @@ def _infer_url(identifier=None): if "ahn3" in identifier: url = "https://service.pdok.nl/rws/ahn3/wcs/v1_0?service=wcs" else: - ValueError(f"unknown identifier -> {identifier}") + raise ValueError(f"unknown identifier -> {identifier}") return url @@ -241,6 +241,7 @@ def get_ahn4_tiles(extent=None): return gdf +@cache.cache_netcdf def get_ahn1(extent, identifier="ahn1_5m", as_data_array=True): """Download AHN1. @@ -267,6 +268,7 @@ def get_ahn1(extent, identifier="ahn1_5m", as_data_array=True): return da +@cache.cache_netcdf def get_ahn2(extent, identifier="ahn2_5m", as_data_array=True): """Download AHN2. @@ -290,6 +292,7 @@ def get_ahn2(extent, identifier="ahn2_5m", as_data_array=True): return _download_and_combine_tiles(tiles, identifier, extent, as_data_array) +@cache.cache_netcdf def get_ahn3(extent, identifier="AHN3_5m_DTM", as_data_array=True): """Download AHN3. @@ -312,6 +315,7 @@ def get_ahn3(extent, identifier="AHN3_5m_DTM", as_data_array=True): return _download_and_combine_tiles(tiles, identifier, extent, as_data_array) +@cache.cache_netcdf def get_ahn4(extent, identifier="AHN4_DTM_5m", as_data_array=True): """Download AHN4. diff --git a/nlmod/read/bgt.py b/nlmod/read/bgt.py index ec6d3801..db7b2ef9 100644 --- a/nlmod/read/bgt.py +++ b/nlmod/read/bgt.py @@ -1,7 +1,7 @@ import json import time -import xml.etree.ElementTree as ET from io import BytesIO +from xml.etree import ElementTree from zipfile import ZipFile import geopandas as gpd @@ -246,7 +246,7 @@ def read_label(child, d): d["label_plaatsingspunt"] = Point(xy) d["label_hoek"] = float(positie.find(f"{ns}hoek").text) - tree = ET.parse(fname) + tree = ElementTree.parse(fname) ns = "{http://www.opengis.net/citygml/2.0}" data = [] for com in tree.findall(f".//{ns}cityObjectMember"): @@ -286,7 +286,7 @@ def read_label(child, d): elif child[0].tag == f"{ns}Point": d[key] = Point(read_point(child[0])) else: - raise (Exception((f"Unsupported tag: {child[0].tag}"))) + raise (ValueError((f"Unsupported tag: {child[0].tag}"))) elif key == "nummeraanduidingreeks": ns = "{http://www.geostandaarden.nl/imgeo/2.1}" nar = child.find(f"{ns}Nummeraanduidingreeks").find( @@ -301,11 +301,11 @@ def read_label(child, d): elif child[0].tag == f"{ns}Curve": d[key] = LineString(read_curve(child[0])) else: - raise (Exception((f"Unsupported tag: {child[0].tag}"))) + raise (ValueError((f"Unsupported tag: {child[0].tag}"))) elif key == "openbareRuimteNaam": read_label(child, d) else: - raise (Exception((f"Unknown key: {key}"))) + raise (KeyError((f"Unknown key: {key}"))) data.append(d) if len(data) > 0: if geometry is None: diff --git a/nlmod/read/boundaries.py b/nlmod/read/boundaries.py new file mode 100644 index 00000000..a7c36c46 --- /dev/null +++ b/nlmod/read/boundaries.py @@ -0,0 +1,68 @@ +from . import waterboard, webservices + + +def get_municipalities(source="cbs", drop_water=True, **kwargs): + """Get the location of the Dutch municipalities as a Polygon GeoDataFrame.""" + if source == "kadaster": + url = ( + "https://service.pdok.nl/kadaster/bestuurlijkegebieden/wfs/v1_0?service=WFS" + ) + layer = "Gemeentegebied" + gdf = webservices.wfs(url, layer, **kwargs) + gdf = gdf.set_index("naam") + elif source == "cbs": + # more course: + # url = "https://service.pdok.nl/cbs/gebiedsindelingen/2023/wfs/v1_0?service=WFS" + # layer = "gemeente_gegeneraliseerd" + # more detail: + url = "https://service.pdok.nl/cbs/wijkenbuurten/2022/wfs/v1_0?&service=WFS" + layer = "gemeenten" + + gdf = webservices.wfs(url, layer, **kwargs) + if drop_water: + gdf = gdf[gdf["water"] == "NEE"] + gdf = gdf.set_index("gemeentenaam") + else: + raise ValueError(f"Unknown source: {source}") + return gdf + + +def get_provinces(source="cbs", **kwargs): + """Get the location of the Dutch provinces as a Polygon GeoDataFrame.""" + if source == "kadaster": + url = ( + "https://service.pdok.nl/kadaster/bestuurlijkegebieden/wfs/v1_0?service=WFS" + ) + layer = "Provinciegebied" + gdf = webservices.wfs(url, layer, **kwargs) + gdf = gdf.set_index("naam") + elif source == "cbs": + url = "https://service.pdok.nl/cbs/gebiedsindelingen/2023/wfs/v1_0?service=WFS" + layer = "provincie_gegeneraliseerd" + gdf = webservices.wfs(url, layer, **kwargs) + gdf = gdf.set_index("statnaam") + else: + raise (ValueError(f"Unknown source: {source}")) + return gdf + + +def get_netherlands(source="cbs", **kwargs): + """Get the location of the Dutch border as a Polygon GeoDataFrame.""" + if source == "kadaster": + url = ( + "https://service.pdok.nl/kadaster/bestuurlijkegebieden/wfs/v1_0?service=WFS" + ) + layer = "Landgebied" + gdf = webservices.wfs(url, layer, **kwargs) + gdf = gdf.set_index("naam") + else: + url = "https://service.pdok.nl/cbs/gebiedsindelingen/2023/wfs/v1_0?service=WFS" + layer = "landsdeel_gegeneraliseerd" + gdf = webservices.wfs(url, layer, **kwargs) + gdf = gdf.set_index("statnaam") + return gdf + + +def get_waterboards(**kwargs): + """Get the location of the Dutch Waterboards as a Polygon GeoDataFrame.""" + return waterboard.get_polygons(**kwargs) diff --git a/nlmod/read/bro.py b/nlmod/read/bro.py new file mode 100644 index 00000000..fe19dbbf --- /dev/null +++ b/nlmod/read/bro.py @@ -0,0 +1,247 @@ +import logging + +import flopy +import hydropandas as hpd +import numpy as np +from shapely.geometry import Point + +from .. import cache, dims, gwf + +logger = logging.getLogger(__name__) + + +def add_modelled_head(oc, ml=None, ds=None, method="linear"): + """add modelled heads as seperate observations to the ObsCollection. + + Parameters + ---------- + oc : ObsCollection + Set of observed groundwater heads + ml : flopy.modflow.mf.model, optional + modflow model, by default None + ds : xr.DataSet, optional + dataset with relevant model information, by default None + method : str, optional + type of interpolation used to get heads. For now only 'linear' and + 'nearest' are supported. The default is 'linear'. + + Returns + ------- + ObsCollection + combination of observed and modelled groundwater heads. + """ + + oc["modellayer"] = oc.gwobs.get_modellayers(gwf=ml) + if ds is not None and "heads" in ds: + heads = ds["heads"] + else: + heads = gwf.get_heads_da(ds=ds, gwf=ml) + + # this function requires a flopy model object, see + # https://github.com/ArtesiaWater/hydropandas/issues/146 + oc_modflow = hpd.read_modflow(oc, ml, heads.values, ds.time.values, method=method) + + if ds.gridtype == "vertex": + gi = flopy.utils.GridIntersect(dims.modelgrid_from_ds(ds), method="vertex") + + obs_list = [] + for name in oc.index: + o = oc.loc[name, "obs"] + modellayer = oc.loc[name, "modellayer"] + if "qualifier" in o.columns: + o = o[o["qualifier"] == "goedgekeurd"] + o_resampled = o.resample("D").last().sort_index() + modelled = oc_modflow.loc[name, "obs"] + + if ds.gridtype == "structured": + bot = ds["botm"].interp(x=o.x, y=o.y, method="nearest").values[modellayer] + if modellayer == 0: + top = ds["top"].interp(x=o.x, y=o.y, method="nearest") + else: + top = ( + ds["botm"] + .interp(x=o.x, y=o.y, method="nearest") + .values[modellayer - 1] + ) + elif ds.gridtype == "vertex": + icelld2 = gi.intersect(Point(o.x, o.y))["cellids"][0] + bot = ds["botm"].values[modellayer, icelld2] + if modellayer == 0: + top = ds["top"].values[icelld2] + else: + top = ds["botm"].values[modellayer - 1, icelld2] + else: + raise ValueError("unexpected gridtype") + + modelled = hpd.GroundwaterObs( + modelled.rename(columns={0: "values"}), + name=f"{o.name}_model", + x=o.x, + y=o.y, + tube_nr=o.tube_nr, + screen_top=top, + screen_bottom=bot, + monitoring_well=o.monitoring_well, + source="MODFLOW", + unit="m NAP", + metadata_available=o.metadata_available, + ) + obs_list.append(o_resampled) + obs_list.append(modelled) + oc_compare = hpd.ObsCollection(obs_list, name="meting+model") + + return oc_compare + + +@cache.cache_pickle +def get_bro( + extent, + regis_layers=None, + max_screen_top=None, + min_screen_bot=None, +): + """get bro groundwater measurements within an extent. + + Parameters + ---------- + extent : list or tuple + get bro groundwater measurements within this extent, + (xmin, xmax, ymin, ymax). + regis_layers : str, list or tuple, optional + get only measurements within these regis layers, by default None + max_screen_top : int or float, optional + get only measurements with a screen top lower than this, by default None + min_screen_bot : int or float, optional + get only measurements with a screen bottom higher than this, by default + None. + + Returns + ------- + ObsCollection + obsevations + """ + oc_meta = get_bro_metadata(extent) + oc_meta = oc_meta.loc[~oc_meta["gld_ids"].isna()] + if oc_meta.empty: + logger.warning("none of the observation wells have measurements") + return oc_meta + + if regis_layers is not None: + if isinstance(regis_layers, str): + regis_layers = [regis_layers] + + oc_meta["regis_layer"] = oc_meta.gwobs.get_regis_layers() + oc_meta = oc_meta[oc_meta["regis_layer"].isin(regis_layers)] + if oc_meta.empty: + logger.warning( + f"none of the regis layers {regis_layers} found in the observation wells" + ) + return oc_meta + + if max_screen_top is not None: + oc_meta = oc_meta[oc_meta["screen_top"] < max_screen_top] + if oc_meta.empty: + logger.warning( + f"none of the observation wells have a screen lower than {max_screen_top}" + ) + return oc_meta + + if min_screen_bot is not None: + oc_meta = oc_meta[oc_meta["screen_bottom"] > min_screen_bot] + if oc_meta.empty: + logger.warning( + f"none of the observation wells have a screen higher than {min_screen_bot}" + ) + return oc_meta + + # download measurements + new_obs_list = [] + for gld_ids in oc_meta["gld_ids"]: + for gld_id in gld_ids: + new_o = hpd.GroundwaterObs.from_bro(gld_id) + if not new_o.empty: + new_obs_list.append(new_o) + oc = hpd.ObsCollection.from_list(new_obs_list) + + return oc + + +@cache.cache_pickle +def get_bro_metadata(extent, max_dx=20000, max_dy=20000): + """wrapper around hpd.read_bro that deals with large extents and only + returns metadata (location, tube top/bot, ground level, ..) of the wells + and no actual measurements. This is useful when the extent is too big + to obtain all measurements at once. + + Parameters + ---------- + extent : list, tuple or np.array + desired model extent (xmin, xmax, ymin, ymax) + max_dx : int, optional + maximum distance in y direction that can be downloaded at once, by + default 20000 meters (20 km) + max_dy : int, optional + maximum distance in x direction that can be downloaded at once, by + default 20000 meters (20 km) + + Returns + ------- + ObsCollection + """ + + # check if extent is within limits + dx = extent[1] - extent[0] + dy = extent[3] - extent[2] + + # check if size exceeds maxsize + if dx > max_dx: + x_segments = int(np.ceil(dx / max_dx)) + else: + x_segments = 1 + + if dy > max_dy: + y_segments = int(np.ceil(dy / max_dy)) + else: + y_segments = 1 + + if (x_segments * y_segments) > 1: + st = f"""requested bro dataset width or height bigger than {max_dx} or {max_dy} + -> splitting extent into {x_segments} * {y_segments} tiles""" + logger.info(st) + d = {} + + for tx in range(x_segments): + for ty in range(y_segments): + xmin = extent[0] + tx * max_dx + xmax = min(extent[1], extent[0] + (tx + 1) * max_dx) + ymin = extent[2] + ty * max_dy + ymax = min(extent[3], extent[2] + (ty + 1) * max_dx) + logger.debug( + f"reading bro within extent {xmin}, {xmax}, {ymin}, {ymax}" + ) + oc = hpd.read_bro( + (xmin, xmax, ymin, ymax), + only_metadata=True, + name="BRO", + ignore_max_obs=True, + ) + d[f"{tx}_{ty}"] = oc + + # merge datasets + i = 0 + for item in d.values(): + if i == 0: + oc = item + else: + if not item.empty: + oc = oc.add_obs_collection(item) + i += 1 + else: + oc = hpd.read_bro(extent, only_metadata=True, name="BRO", ignore_max_obs=True) + + oc.add_meta_to_df("gld_ids") + + if oc.empty: + logger.warning("no observation wells within extent") + + return oc diff --git a/nlmod/read/brp.py b/nlmod/read/brp.py index 8628836c..7824df4e 100644 --- a/nlmod/read/brp.py +++ b/nlmod/read/brp.py @@ -10,7 +10,7 @@ def get_percelen(extent, year=None): gdf = gdf.set_index("fuuid") else: if year < 2009 or year > 2021: - raise (Exception("Only data available from 2009 up to and including 2021")) + raise (ValueError("Only data available from 2009 up to and including 2021")) url = f"https://services.arcgis.com/nSZVuSZjHpEZZbRo/ArcGIS/rest/services/BRP_{year}/FeatureServer" gdf = webservices.arcrest(url, 0, extent=extent) return gdf diff --git a/nlmod/read/geotop.py b/nlmod/read/geotop.py index dfcb574a..d4e2e425 100644 --- a/nlmod/read/geotop.py +++ b/nlmod/read/geotop.py @@ -1,12 +1,15 @@ import datetime as dt import logging import os +import warnings import numpy as np import pandas as pd import xarray as xr from .. import NLMOD_DATADIR, cache +from ..dims.layers import insert_layer, remove_layer +from ..util import MissingValueError logger = logging.getLogger(__name__) @@ -33,7 +36,7 @@ def get_lithok_colors(): 8: (216, 163, 32), 9: (95, 95, 255), } - colors = {key: tuple([x / 255 for x in colors[key]]) for key in colors} + colors = {key: tuple(x / 255 for x in color) for key, color in colors.items()} return colors @@ -52,37 +55,160 @@ def get_kh_kv_table(kind="Brabant"): ) df = pd.read_csv(fname) else: - raise (Exception(f"Unknown kind in get_kh_kv_table: {kind}")) + raise (ValueError(f"Unknown kind in get_kh_kv_table: '{kind}'")) return df @cache.cache_netcdf -def get_geotop(extent, strat_props=None): - """get a model layer dataset for modflow from geotop within a certain - extent and grid. +def to_model_layers( + geotop_ds, strat_props=None, method_geulen="add_to_layer_below", **kwargs +): + """Convert geotop voxel dataset to layered dataset. + + Converts geotop data to dataset with layer elevations and hydraulic conductivities. + Optionally uses hydraulic conductivities provided present in geotop_ds. Parameters ---------- - extent : list, tuple or np.array - desired model extent (xmin, xmax, ymin, ymax) + geotop_ds : xr.DataSet + geotop voxel dataset (download using `get_geotop(extent)`) strat_props : pd.DataFrame, optional - The properties of the stratigraphic unit. Load with get_strat_props() when None. - The default is None. + The properties (code and name) of the stratigraphic units. Load with + get_strat_props() when None. The default is None. + method_geulen : str, optional + strat-units >=6000 are so-called 'geulen' (paleochannels, gullies). These are + difficult to add to the layer model, because they can occur above and/or below + any other unit. Multiple methods are available to handle these 'geulen'. + The method "add_to_layer_below" adds the thickness of the 'geul' to the layer + with a positive thickness below the 'geul'. The method "add_to_layer_above" + adds the thickness of the 'geul' to the layer with a positive thickness above + the 'geul'. The method "add_as_layer" tries to add the 'geulen' as one or more + layers, which can fail if a 'geul' is locally both below the top and above the + bottom of another layer (splitting the layer in two, which is not supported). + The default is "add_to_layer_below". + kwargs : dict + Kwargs are passed to `aggregate_to_ds()` Returns ------- - geotop_ds: xr.DataSet - geotop dataset with top, bot, kh and kv per geo_eenheid + ds: xr.DataSet + dataset with top and botm (and optionally kh and kv) per geotop layer """ - gt = get_geotop_raw_within_extent(extent, GEOTOP_URL) if strat_props is None: strat_props = get_strat_props() - ds = convert_geotop_to_ml_layers(gt, strat_props=strat_props) + # stap 2 create layer for each stratigraphy unit (geo-eenheid) + if strat_props is None: + strat_props = get_strat_props() + + # get all strat-units in Dataset + strat = geotop_ds["strat"].values + units = np.unique(strat) + units = units[~np.isnan(units)].astype(int) + shape = (len(units), len(geotop_ds.y), len(geotop_ds.x)) + + # stratigraphy unit (geo eenheid) 2000 is above 1130 + if (2000 in units) and (1130 in units): + units[(units == 2000) + (units == 1130)] = [2000, 1130] + + # fill top and bot + top = np.full(shape, np.nan) + bot = np.full(shape, np.nan) + + z = ( + geotop_ds["z"] + .data[:, np.newaxis, np.newaxis] + .repeat(len(geotop_ds.y), 1) + .repeat(len(geotop_ds.x), 2) + ) + layers = [] + geulen = [] + for layer, unit in enumerate(units): + mask = strat == unit + top[layer] = np.nanmax(np.where(mask, z, np.NaN), 0) + 0.25 + bot[layer] = np.nanmin(np.where(mask, z, np.NaN), 0) - 0.25 + if int(unit) in strat_props.index: + layers.append(strat_props.at[unit, "code"]) + else: + logger.warning(f"Unknown strat-value: {unit}") + layers.append(str(unit)) + if unit >= 6000: + geulen.append(layers[-1]) + + dims = ("layer", "y", "x") + coords = {"layer": layers, "y": geotop_ds.y, "x": geotop_ds.x} + ds = xr.Dataset({"top": (dims, top), "botm": (dims, bot)}, coords=coords) + + if method_geulen is None: + pass + elif method_geulen == "add_as_layer": + top = ds["top"].copy(deep=True) + bot = ds["botm"].copy(deep=True) + for geul in geulen: + ds = remove_layer(ds, geul) + for geul in geulen: + ds = insert_layer(ds, geul, top.loc[geul], bot.loc[geul]) + elif method_geulen == "add_to_layer_below": + top = ds["top"].copy(deep=True) + bot = ds["botm"].copy(deep=True) + for geul in geulen: + ds = remove_layer(ds, geul) + for geul in geulen: + todo = (top.loc[geul] - bot.loc[geul]) > 0.0 + for layer in ds.layer: + if not todo.any(): + continue + # adds the thickness of the geul to the layer below the geul + mask = (top.loc[geul] > bot.loc[layer]) & todo + if mask.any(): + ds["top"].loc[layer].data[mask] = np.maximum( + top.loc[geul].data[mask], top.loc[layer].data[mask] + ) + todo.data[mask] = False + if todo.any(): + # unless the geul is the bottom layer + # then its thickness is added to the last active layer + # idomain = get_idomain(ds) + # fal = get_last_active_layer_from_idomain(idomain) + logger.warning( + f"Geul {geul} is at the bottom of the GeoTOP-dataset in {int(todo.sum())} cells, where it is ignored" + ) + + elif method_geulen == "add_to_layer_above": + top = ds["top"].copy(deep=True) + bot = ds["botm"].copy(deep=True) + for geul in geulen: + ds = remove_layer(ds, geul) + for geul in geulen: + todo = (top.loc[geul] - bot.loc[geul]) > 0.0 + for layer in reversed(ds.layer): + if not todo.any(): + continue + # adds the thickness of the geul to the layer above the geul + mask = (bot.loc[geul] < top.loc[layer]) & todo + if mask.any(): + ds["botm"].loc[layer].data[mask] = np.minimum( + bot.loc[geul].data[mask], bot.loc[layer].data[mask] + ) + todo.data[mask] = False + if todo.any(): + # unless the geul is the top layer + # then its thickness is added to the last active layer + # idomain = get_idomain(ds) + # fal = get_first_active_layer_from_idomain(idomain) + logger.warning( + f"Geul {geul} is at the top of the GeoTOP-dataset in {int(todo.sum())} cells, where it is ignored" + ) + else: + raise (ValueError(f"Unknown method to deal with geulen: '{method_geulen}'")) + + ds.attrs["geulen"] = geulen - ds.attrs["extent"] = extent + if "kh" in geotop_ds and "kv" in geotop_ds: + aggregate_to_ds(geotop_ds, ds, **kwargs) + # add atributes for datavar in ds: ds[datavar].attrs["source"] = "Geotop" ds[datavar].attrs["url"] = GEOTOP_URL @@ -95,7 +221,8 @@ def get_geotop(extent, strat_props=None): return ds -def get_geotop_raw_within_extent(extent, url=GEOTOP_URL, drop_probabilities=True): +@cache.cache_netcdf +def get_geotop(extent, url=GEOTOP_URL, probabilities=False): """Get a slice of the geotop netcdf url within the extent, set the x and y coordinates to match the cell centers and keep only the strat and lithok data variables. @@ -107,128 +234,59 @@ def get_geotop_raw_within_extent(extent, url=GEOTOP_URL, drop_probabilities=True url : str, optional url of geotop netcdf file. The default is http://www.dinodata.nl/opendap/GeoTOP/geotop.nc + probabilities : bool, optional + if True, download probability data Returns ------- gt : xarray Dataset slices geotop netcdf. """ - gt = xr.open_dataset(url) + gt = xr.open_dataset(url, chunks="auto") + + # only download requisite data + data_vars = ["strat", "lithok"] + if probabilities: + data_vars += [ + "kans_1", + "kans_2", + "kans_3", + "kans_4", + "kans_5", + "kans_6", + "kans_7", + "kans_8", + "kans_9", + "onz_lk", + "onz_ls", + ] # set x and y dimensions to cell center for dim in ["x", "y"]: old_dim = gt[dim].values gt[dim] = old_dim + (old_dim[1] - old_dim[0]) / 2 - # slice extent - gt = gt.sel(x=slice(extent[0], extent[1]), y=slice(extent[2], extent[3])) - - if drop_probabilities: - gt = gt[["strat", "lithok"]] + # get data vars and slice extent + gt = gt[data_vars].sel(x=slice(extent[0], extent[1]), y=slice(extent[2], extent[3])) # change order of dimensions from x, y, z to z, y, x gt = gt.transpose("z", "y", "x") - gt = gt.sortby("z", ascending=False) # uses a lot of RAM - gt = gt.sortby("y", ascending=False) # uses a lot of RAM - - return gt - -def convert_geotop_to_ml_layers( - geotop_ds_raw, - strat_props=None, - **kwargs, -): - """Convert geotop voxel data to layers using the stratigraphy-data. - - It gets the top and botm of each stratigraphic unit in the geotop dataset. - - Parameters - ---------- - geotop_ds_raw: xr.Dataset - dataset with geotop voxel data - strat_props: pandas.DataFrame - The properties of the stratigraphic unit. Load with get_strat_props() when None. - The default is None. - - Returns - ------- - geotop_ds_mod: xarray.DataSet - geotop dataset with top and botm per geo_eenheid - - Note - ---- - strat-units >=6000 are 'stroombanen'. These are difficult to add because they can - occur above and/or below any other unit. Therefore these units are not added to the - dataset, and their thickness is added to the strat-unit below the stroombaan. - """ - - # stap 2 maak een laag per geo-eenheid - if strat_props is None: - strat_props = get_strat_props() - - # vindt alle geo-eenheden in model_extent - geo_eenheden = np.unique(geotop_ds_raw.strat.data) - geo_eenheden = geo_eenheden[np.isfinite(geo_eenheden)] - stroombaan_eenheden = geo_eenheden[geo_eenheden >= 6000] - geo_eenheden = geo_eenheden[geo_eenheden < 6000] - - # geo eenheid 2000 zit boven 1130 - if (2000.0 in geo_eenheden) and (1130.0 in geo_eenheden): - geo_eenheden[(geo_eenheden == 2000.0) + (geo_eenheden == 1130.0)] = [ - 2000.0, - 1130.0, - ] - - strat_codes = [] - for geo_eenh in geo_eenheden: - if int(geo_eenh) in strat_props.index: - code = strat_props.at[int(geo_eenh), "code"] - else: - logger.warning(f"Unknown strat-value: {geo_eenh}") - code = str(geo_eenh) - strat_codes.append(code) - - # fill top and bot - shape = (len(strat_codes), len(geotop_ds_raw.y), len(geotop_ds_raw.x)) - top = np.full(shape, np.nan) - bot = np.full(shape, np.nan) - lay = 0 - logger.info("creating top and bot per geo eenheid") - for geo_eenheid in geo_eenheden: - logger.debug(int(geo_eenheid)) - - mask = geotop_ds_raw.strat == geo_eenheid - geo_z = xr.where(mask, geotop_ds_raw.z, np.NaN) + # flip z, and y coordinates + gt = gt.isel(z=slice(None, None, -1), y=slice(None, None, -1)) - top[lay] = geo_z.max(dim="z") + 0.25 - bot[lay] = geo_z.min(dim="z") - 0.25 + # add missing value + # gt.strat.attrs["missing_value"] = -127 - lay += 1 - - # add the thickness of stroombanen to the layer below the stroombaan - for lay in range(top.shape[0]): - if lay == 0: - top[lay] = np.nanmax(top, 0) - else: - top[lay] = bot[lay - 1] - bot[lay] = np.where(np.isnan(bot[lay]), top[lay], bot[lay]) - - dims = ("layer", "y", "x") - coords = {"layer": strat_codes, "y": geotop_ds_raw.y, "x": geotop_ds_raw.x} - da_top = xr.DataArray(data=top, dims=dims, coords=coords) - da_bot = xr.DataArray(data=bot, dims=dims, coords=coords) - geotop_ds_mod = xr.Dataset() - - geotop_ds_mod["top"] = da_top - geotop_ds_mod["botm"] = da_bot - - geotop_ds_mod.attrs["stroombanen"] = stroombaan_eenheden + return gt - if "kh" in geotop_ds_raw and "kv" in geotop_ds_raw: - aggregate_to_ds(geotop_ds_raw, geotop_ds_mod, **kwargs) - return geotop_ds_mod +def get_geotop_raw_within_extent(extent, url=GEOTOP_URL, drop_probabilities=True): + warnings.warn( + "This function is deprecated, use the equivalent `get_geotop()`!", + DeprecationWarning, + ) + return get_geotop(extent=extent, url=url, probabilities=not drop_probabilities) def add_top_and_botm(ds): @@ -247,18 +305,14 @@ def add_top_and_botm(ds): ds : xr.Dataset The geotop-dataset, with added variables "top" and "botm". """ - # make ready for DataSetCrossSection - # ds = ds.transpose("z", "y", "x") - # ds = ds.sortby("z", ascending=False) - - bottom = np.expand_dims(ds.z.data - 0.25, axis=(1, 2)) + bottom = np.expand_dims(ds.z.values - 0.25, axis=(1, 2)) bottom = np.repeat(np.repeat(bottom, len(ds.y), 1), len(ds.x), 2) - bottom[np.isnan(ds.strat.data)] = np.NaN + bottom[np.isnan(ds.strat.values)] = np.NaN ds["botm"] = ("z", "y", "x"), bottom - top = np.expand_dims(ds.z.data + 0.25, axis=(1, 2)) + top = np.expand_dims(ds.z.values + 0.25, axis=(1, 2)) top = np.repeat(np.repeat(top, len(ds.y), 1), len(ds.x), 2) - top[np.isnan(ds.strat.data)] = np.NaN + top[np.isnan(ds.strat.values)] = np.NaN ds["top"] = ("z", "y", "x"), top return ds @@ -306,7 +360,7 @@ def add_kh_and_kv( kh : str, optional THe name of the new variable with kh values in gt. The default is "kh". kv : str, optional - THe name of the new variable with kv values in gt. The default is "kv". + The name of the new variable with kv values in gt. The default is "kv". kh_df : str, optional The name of the column with kh values in df. The default is "kh". kv_df : str, optional @@ -328,10 +382,10 @@ def add_kh_and_kv( else: stochastic = None if kh_method not in ["arithmetic_mean", "harmonic_mean"]: - raise (Exception("Unknown kh_method: {kh_method}")) + raise (ValueError("Unknown kh_method: {kh_method}")) if kv_method not in ["arithmetic_mean", "harmonic_mean"]: - raise (Exception("Unknown kv_method: {kv_method}")) - strat = gt["strat"].data + raise (ValueError("Unknown kv_method: {kv_method}")) + strat = gt["strat"].values msg = "Determining kh and kv of geotop-data based on lithoclass" if df.index.name in ["lithok", "strat"]: df = df.reset_index() @@ -339,12 +393,12 @@ def add_kh_and_kv( msg = f"{msg} and stratigraphy" logger.info(msg) if kh_df not in df: - raise (Exception(f"No {kh_df} defined in df")) + raise (MissingValueError(f"No {kh_df} defined in df")) if kv_df not in df: logger.info(f"Setting kv equal to kh / {anisotropy}") if stochastic is None: # calculate kh and kv from most likely lithoclass - lithok = gt["lithok"].data + lithok = gt["lithok"].values kh_ar = np.full(lithok.shape, np.NaN) kv_ar = np.full(lithok.shape, np.NaN) if "strat" in df: @@ -376,7 +430,7 @@ def add_kh_and_kv( if ilithok == 0: # there are no probabilities defined for lithoclass 'antropogeen' continue - probality = gt[f"kans_{ilithok}"].data + probality = gt[f"kans_{ilithok}"].values if "strat" in df: khi, kvi = _handle_nans_in_stochastic_approach( np.NaN, np.NaN, kh_method, kv_method @@ -421,7 +475,7 @@ def add_kh_and_kv( else: kv_ar = probality_total / kv_ar else: - raise (Exception(f"Unsupported value for stochastic: {stochastic}")) + raise (ValueError(f"Unsupported value for stochastic: '{stochastic}'")) dims = gt["strat"].dims gt[kh] = dims, kh_ar @@ -510,21 +564,20 @@ def aggregate_to_ds( The Dataset ds, with added variables kh and kv (and optionally kd and c). """ assert (ds.x == gt.x).all() and (ds.y == gt.y).all() - msg = "Please add {} to geotop-Dataset first, using add_kh_and_kv()" + msg = "Please add '{}' to geotop-Dataset first, using add_kh_and_kv()" if kh_gt not in gt: - raise (Exception(msg.format(kh_gt))) + raise (MissingValueError(msg.format(kh_gt))) if kv_gt not in gt: - raise (Exception(msg.format(kv_gt))) + raise (MissingValueError(msg.format(kv_gt))) kD_ar = [] c_ar = [] kh_ar = [] kv_ar = [] - if "layer" in ds["top"].dims: - # make sure there is no layer dimension in top - ds["top"] = ds["top"].max("layer") for ilay in range(len(ds.layer)): if ilay == 0: top = ds["top"] + if "layer" in top.dims: + top = top[0].drop_vars("layer") else: top = ds["botm"][ilay - 1].drop_vars("layer") bot = ds["botm"][ilay].drop_vars("layer") diff --git a/nlmod/read/jarkus.py b/nlmod/read/jarkus.py index 41147e98..93e914e4 100644 --- a/nlmod/read/jarkus.py +++ b/nlmod/read/jarkus.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """module with functions to deal with the northsea by: - identifying model cells with the north sea @@ -18,7 +17,7 @@ import xarray as xr from .. import cache -from ..dims.resample import fillnan_da, structured_da_to_ds +from ..dims.resample import fillnan_da, get_extent, structured_da_to_ds from ..util import get_da_from_da_ds, get_ds_empty logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ def get_bathymetry(ds, northsea, kind="jarkus", method="average"): data is resampled to the modelgrid. Maybe we can speed up things by changing the order in which operations are executed. """ - ds_out = get_ds_empty(ds) + ds_out = get_ds_empty(ds, keep_coords=("y", "x")) # no bathymetry if we don't have northsea if (northsea == 0).all(): @@ -59,7 +58,7 @@ def get_bathymetry(ds, northsea, kind="jarkus", method="average"): # try to get bathymetry via opendap try: - jarkus_ds = get_dataset_jarkus(ds.extent, kind=kind) + jarkus_ds = get_dataset_jarkus(get_extent(ds), kind=kind) except OSError: import gdown @@ -93,6 +92,7 @@ def get_bathymetry(ds, northsea, kind="jarkus", method="average"): return ds_out +@cache.cache_netcdf def get_dataset_jarkus(extent, kind="jarkus", return_tiles=False, time=-1): """Get bathymetry from Jarkus within a certain extent. If return_tiles is False, the following actions are performed: @@ -105,7 +105,7 @@ def get_dataset_jarkus(extent, kind="jarkus", return_tiles=False, time=-1): ---------- extent : list, tuple or np.array extent (xmin, xmax, ymin, ymax) of the desired grid. Should be RD-new - coördinates (EPSG:28992) + coordinates (EPSG:28992) kind : str, optional The kind of data. Can be "jarkus", "kusthoogte" or "vaklodingen". The default is "jarkus". @@ -163,6 +163,8 @@ def get_dataset_jarkus(extent, kind="jarkus", return_tiles=False, time=-1): ) tiles = tiles_left z_dataset = xr.combine_by_coords(tiles, combine_attrs="drop") + # drop 'lat' and 'lon' as these will create problems when resampling the data + z_dataset = z_dataset.drop_vars(["lat", "lon"]) return z_dataset @@ -173,7 +175,7 @@ def get_jarkus_tilenames(extent, kind="jarkus"): ---------- extent : list, tuple or np.array extent (xmin, xmax, ymin, ymax) of the desired grid. Should be RD-new - coördinates (EPSG:28992) + coordinates (EPSG:28992) Returns ------- @@ -187,7 +189,7 @@ def get_jarkus_tilenames(extent, kind="jarkus"): elif kind == "vaklodingen": url = "http://opendap.deltares.nl/thredds/dodsC/opendap/rijkswaterstaat/vaklodingen/catalog.nc" else: - raise (Exception(f"Unsupported kind: {kind}")) + raise (ValueError(f"Unsupported kind: {kind}")) ds_jarkus_catalog = xr.open_dataset(url) ew_x = ds_jarkus_catalog["projectionCoverage_x"].values @@ -238,8 +240,12 @@ def get_netcdf_tiles(kind="jarkus"): def add_bathymetry_to_top_bot_kh_kv(ds, bathymetry, fill_mask, kh_sea=10, kv_sea=10): - """add bathymetry to the top and bot of each layer for all cells with - fill_mask. + """Add bathymetry to the top and bot of each layer for all cells with fill_mask. + + This method sets the top of the model at fill_mask to 0 m, and changes the first + layer to sea, by setting the botm of this layer to bathymetry, kh to kh_sea and kv + to kv_sea. If deeper layers are above bathymetry. the layer depth is set to + bathymetry. Parameters ---------- diff --git a/nlmod/read/knmi.py b/nlmod/read/knmi.py index 1abe66b9..9499817f 100644 --- a/nlmod/read/knmi.py +++ b/nlmod/read/knmi.py @@ -38,7 +38,7 @@ def get_recharge(ds, method="linear", most_common_station=False): ---------- ds : xr.DataSet dataset containing relevant model grid information - method : bool, optional + method : str, optional If 'linear', calculate recharge by subtracting evaporation from precipitation. If 'separate', add precipitation as 'recharge' and evaporation as 'evaporation'. The defaults is 'linear'. @@ -62,7 +62,7 @@ def get_recharge(ds, method="linear", most_common_station=False): start = pd.Timestamp(ds.time.attrs["start"]) end = pd.Timestamp(ds.time.data[-1]) - ds_out = util.get_ds_empty(ds) + ds_out = util.get_ds_empty(ds, keep_coords=("time", "y", "x")) ds_out.attrs["gridtype"] = ds.gridtype # get recharge data array @@ -85,7 +85,9 @@ def get_recharge(ds, method="linear", most_common_station=False): unique_combinations = locations.drop_duplicates(["stn_rd", "stn_ev24"])[ ["stn_rd", "stn_ev24"] ].values - + if unique_combinations.shape[1] > 2: + # bug fix for pandas 2.1 where three columns are returned + unique_combinations = unique_combinations[:, :2] for stn_rd, stn_ev24 in unique_combinations: # get locations with the same prec and evap station mask = (locations["stn_rd"] == stn_rd) & (locations["stn_ev24"] == stn_ev24) @@ -116,7 +118,7 @@ def get_recharge(ds, method="linear", most_common_station=False): loc_sel = locations.loc[(locations["stn_ev24"] == stn)] _add_ts_to_ds(ts, loc_sel, "evaporation", ds_out) else: - raise (Exception(f"Unknown method: {method}")) + raise (ValueError(f"Unknown method: {method}")) for datavar in ds_out: ds_out[datavar].attrs["source"] = "KNMI" ds_out[datavar].attrs["date"] = dt.datetime.now().strftime("%Y%m%d") @@ -129,7 +131,9 @@ def _add_ts_to_ds(timeseries, loc_sel, variable, ds): """Add a timeseries to a variable at location loc_sel in model DataSet.""" end = pd.Timestamp(ds.time.data[-1]) if timeseries.index[-1] < end: - raise ValueError(f"no recharge available at {timeseries.name} for date {end}") + raise ValueError( + f"no data available for time series'{timeseries.name}' on date {end}" + ) # fill recharge data array model_recharge = pd.Series(index=ds.time, dtype=float) @@ -145,7 +149,7 @@ def _add_ts_to_ds(timeseries, loc_sel, variable, ds): # there will be NaN's, which we fill by backfill model_recharge = model_recharge.fillna(method="bfill") if model_recharge.isna().any(): - raise (Exception("There are NaN-values in {variable}")) + raise (ValueError(f"There are NaN-values in {variable}.")) # add data to ds values = np.repeat(model_recharge.values[:, np.newaxis], loc_sel.shape[0], 1) @@ -171,7 +175,7 @@ def get_locations_vertex(ds): """ # get active locations fal = get_first_active_layer(ds) - icell2d_active = np.where(fal != fal.attrs["_FillValue"])[0] + icell2d_active = np.where(fal != fal.attrs["nodata"])[0] # create dataframe from active locations x = ds["x"].sel(icell2d=icell2d_active) @@ -206,7 +210,7 @@ def get_locations_structured(ds): # store x and y mids in locations of active cells fal = get_first_active_layer(ds) - rows, columns = np.where(fal != fal.attrs["_FillValue"]) + rows, columns = np.where(fal != fal.attrs["nodata"]) x = np.array([ds["x"].data[col] for col in columns]) y = np.array([ds["y"].data[row] for row in rows]) if "angrot" in ds.attrs and ds.attrs["angrot"] != 0.0: @@ -275,11 +279,19 @@ def get_knmi_at_locations(ds, start="2010", end=None, most_common_station=False) # get knmi data stations closest to any grid cell oc_knmi_prec = hpd.ObsCollection.from_knmi( - stns=stns_rd, starts=[start], ends=[end], meteo_vars=["RD"] + stns=stns_rd, + starts=[start], + ends=[end], + meteo_vars=["RD"], + fill_missing_obs=True, ) oc_knmi_evap = hpd.ObsCollection.from_knmi( - stns=stns_ev24, starts=[start], ends=[end], meteo_vars=["EV24"] + stns=stns_ev24, + starts=[start], + ends=[end], + meteo_vars=["EV24"], + fill_missing_obs=True, ) return locations, oc_knmi_prec, oc_knmi_evap diff --git a/nlmod/read/meteobase.py b/nlmod/read/meteobase.py index cf202e6e..a5df07db 100644 --- a/nlmod/read/meteobase.py +++ b/nlmod/read/meteobase.py @@ -231,11 +231,11 @@ def read_meteobase_ascii( da = DataArray( data_array, dims=["time", "y", "x"], - coords=dict( - time=times, - x=x, - y=y, - ), + coords={ + "time": times, + "x": x, + "y": y, + }, attrs=meta, name=foldername, ) diff --git a/nlmod/read/nhi.py b/nlmod/read/nhi.py new file mode 100644 index 00000000..858e3a16 --- /dev/null +++ b/nlmod/read/nhi.py @@ -0,0 +1,175 @@ +import logging +import os + +import numpy as np +import requests +import rioxarray + +from ..dims.resample import structured_da_to_ds + +logger = logging.getLogger(__name__) + + +def download_file(url, pathname, filename=None, overwrite=False, timeout=120.0): + """ + Download a file from the NHI website. + + Parameters + ---------- + url : str + The url of the file to download. + pathname : str + The pathname to which the file is downloaded. + filename : str, optional + The name of the file to contain the downloadded data. When filename is None, it + is derived from url. The default is None. + overwrite : bool, optional + Overwrite the file if it allready exists. The default is False. + timeout : float, optional + How many seconds to wait for the server to send data before giving up. The + default is 120. + + Returns + ------- + fname : str + The full path of the downloaded file. + + """ + if filename is None: + filename = url.split("/")[-1] + fname = os.path.join(pathname, filename) + if overwrite or not os.path.isfile(fname): + logger.info(f"Downloading {filename}") + r = requests.get(url, allow_redirects=True, timeout=timeout) + with open(fname, "wb") as file: + file.write(r.content) + return fname + + +def download_buisdrainage(pathname, overwrite=False): + """ + Download resistance and depth of buisdrainage from the NHI website + + Parameters + ---------- + pathname : str + The pathname to which the files are downloaded. + overwrite : bool, optional + Overwrite the files if they allready exists. The default is False. + + Returns + ------- + fname_c : str + The full path of the downloaded file containing the resistance of buisdrainage. + fname_d : str + The full path of the downloaded file containing the depth of buisdrainage. + + """ + url_bas = "https://thredds.data.nhi.nu/thredds/fileServer/opendap/models/nhi3_2/25m" + + # download resistance + url = f"{url_bas}/buisdrain_c_ras25/buisdrain_c_ras25.nc" + fname_c = download_file(url, pathname, overwrite=overwrite) + + # download drain depth + url = f"{url_bas}/buisdrain_d_ras25/buisdrain_d_ras25.nc" + fname_d = download_file(url, pathname, overwrite=overwrite) + + return fname_c, fname_d + + +def add_buisdrainage( + ds, + pathname=None, + cond_var="buisdrain_cond", + depth_var="buisdrain_depth", + cond_method="average", + depth_method="mode", +): + """ + Add data about the buisdrainage to the model Dataset. + + This data consists of the conductance of buisdrainage (m2/d) and the depth of + buisdrainage (m to surface level). With the default settings for `cond_method` and + `depth_method`, the conductance is calculated from the area weighted average of the + resistance in a cell, while the depth is set to the value which appears most often + in a cell. Cells without data (or 0 in the case of the resistance) are ignored in + these calculations. + + The original data of the resistance and the depth of buisdrainage at a 25x25 m scale + is downloaded to `pathname` when not found. + + Parameters + ---------- + ds : xr.Dataset + The model Dataset. + pathname : str, optional + The pathname containing the downloaded files or the pathname to which the files + are downloaded. When pathname is None, it is set the the cachedir. The default + is None. + cond_var : str, optional + The name of the variable in ds to contain the data about the conductance of + buisdrainage. The default is "buisdrain_cond". + depth_var : str, optional + The name of the variable in ds to contain the data about the depth of + buisdrainage. The default is "buisdrain_depth". + cond_method : str, optional + The method to transform the conductance of buisdrainage to the model Dataset. + The default is "average". + depth_method : str, optional + The method to transform the depth of buisdrainage to the model Dataset. The + default is "mode". + + Returns + ------- + ds : xr.Dataset + The model dataset with added variables with the names `cond_var` and + `depth_var`. + + """ + if pathname is None: + pathname = ds.cachedir + # download files if needed + fname_c, fname_d = download_buisdrainage(pathname) + + # make sure crs is set on ds + if ds.rio.crs is None: + ds = ds.rio.write_crs(28992) + + # use cond_methd for conductance + # (default is "average" to account for locations without pipe drainage, where the + # conductance is 0) + buisdrain_c = rioxarray.open_rasterio(fname_c, mask_and_scale=True)[0] + # calculate a conductance (per m2) from a resistance + cond = 1 / buisdrain_c + # set conductance to 0 where resistance is infinite or 0 + cond = cond.where(~(np.isinf(cond) | np.isnan(cond)), 0.0) + cond = cond.rio.write_crs(buisdrain_c.rio.crs) + # resample to model grid + ds[cond_var] = structured_da_to_ds(cond, ds, method=cond_method) + # multiply by area to get a conductance + ds[cond_var] = ds[cond_var] * ds["area"] + + # use depth_method to retrieve the depth + # (default is "mode" for depth that occurs most in each cell) + mask_and_scale = False + buisdrain_d = rioxarray.open_rasterio(fname_d, mask_and_scale=mask_and_scale)[0] + if mask_and_scale: + nodata = np.nan + else: + nodata = buisdrain_d.attrs["_FillValue"] + # set buisdrain_d to nodata where it is 0 + mask = buisdrain_d != 0 + buisdrain_d = buisdrain_d.where(mask, nodata).rio.write_crs(buisdrain_d.rio.crs) + # resample to model grid + ds[depth_var] = structured_da_to_ds( + buisdrain_d, ds, method=depth_method, nodata=nodata + ) + if not mask_and_scale: + # set nodata values to NaN + ds[depth_var] = ds[depth_var].where(ds[depth_var] != nodata) + + # from cm to m + ds[depth_var] = ds[depth_var] / 100.0 + + return ds diff --git a/nlmod/read/regis.py b/nlmod/read/regis.py index 4a789856..f22419be 100644 --- a/nlmod/read/regis.py +++ b/nlmod/read/regis.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- -"""function to project regis, or a combination of regis and geotop, data on a -modelgrid.""" import datetime as dt import logging import os @@ -77,7 +74,7 @@ def get_combined_layer_models( raise ValueError("layer models without REGIS not supported") if use_geotop: - geotop_ds = geotop.get_geotop_raw_within_extent(extent) + geotop_ds = geotop.get_geotop(extent) if use_regis and use_geotop: combined_ds = add_geotop_to_regis_layers( @@ -139,13 +136,14 @@ def get_regis( if len(ds.x) == 0 or len(ds.y) == 0: msg = "No data found. Please supply valid extent in the Netherlands in RD-coordinates" - raise (Exception(msg)) + raise (ValueError(msg)) # make sure layer names are regular strings ds["layer"] = ds["layer"].astype(str) # make sure y is descending - ds = ds.sortby("y", ascending=False) + if (ds["y"].diff("y") > 0).all(): + ds = ds.isel(y=slice(None, None, -1)) # slice layers if botm_layer is not None: @@ -174,8 +172,6 @@ def get_regis( ds[datavar].attrs["units"] = "mNAP" elif datavar in ["kh", "kv"]: ds[datavar].attrs["units"] = "m/day" - # set _FillValue to NaN, otherise problems with caching will arise - ds[datavar].encoding["_FillValue"] = np.NaN # set the crs to dutch rd-coordinates ds.rio.set_crs(28992) @@ -184,7 +180,7 @@ def get_regis( def add_geotop_to_regis_layers( - rg, gt, layers="HLc", geotop_k=None, remove_nan_layers=True + rg, gt, layers="HLc", geotop_k=None, remove_nan_layers=True, anisotropy=1.0 ): """Combine geotop and regis in such a way that the one or more layers in Regis are replaced by the geo_eenheden of geotop. @@ -202,6 +198,9 @@ def add_geotop_to_regis_layers( DataFrame must at least contain columns 'lithok' and 'kh'. remove_nan_layers : bool, optional When True, layers with only 0 or NaN thickness are removed. The default is True. + anisotropy : float, optional + The anisotropy value (kh/kv) used when there are no kv values in df. The + default is 1.0. Returns ------- @@ -210,43 +209,55 @@ def add_geotop_to_regis_layers( """ if isinstance(layers, str): layers = [layers] - if geotop_k is None: - geotop_k = geotop.get_lithok_props() + + # make sure geotop dataset contains kh and kv + if "kh" not in gt or "kv" not in gt: + if "kv" in gt: + logger.info( + f"Calculating kh of geotop by multiplying kv with an anisotropy of {anisotropy}" + ) + gt["kh"] = gt["kv"] * anisotropy + elif "kh" in gt: + logger.info( + f"Calculating kv of geotop by dividing kh by an anisotropy of {anisotropy}" + ) + gt["kv"] = gt["kh"] / anisotropy + else: + # add kh and kv to gt + if geotop_k is None: + geotop_k = geotop.get_lithok_props() + gt = geotop.add_kh_and_kv(gt, geotop_k, anisotropy=anisotropy) + for layer in layers: # transform geotop data into layers - gtl = geotop.convert_geotop_to_ml_layers(gt) + gtl = geotop.to_model_layers(gt) + + # make sure top is 3d + assert "layer" in rg["top"].dims, "Top of regis must be 3d" + assert "layer" in gtl["top"].dims, "Top of geotop layers must be 3d" # only keep the part of layers inside the regis layer top = rg["top"].loc[layer] bot = rg["botm"].loc[layer] - gtl["top"] = gtl["top"].where(gtl["top"] < top, top) - gtl["top"] = gtl["top"].where(gtl["top"] > bot, bot) - gtl["botm"] = gtl["botm"].where(gtl["botm"] < top, top) - gtl["botm"] = gtl["botm"].where(gtl["botm"] > bot, bot) + gtl["top"] = gtl["top"].where(gtl["top"].isnull() | (gtl["top"] < top), top) + gtl["top"] = gtl["top"].where(gtl["top"].isnull() | (gtl["top"] > bot), bot) + gtl["botm"] = gtl["botm"].where(gtl["botm"].isnull() | (gtl["botm"] < top), top) + gtl["botm"] = gtl["botm"].where(gtl["botm"].isnull() | (gtl["botm"] > bot), bot) if remove_nan_layers: # drop layers with a remaining thickness of 0 (or NaN) everywhere th = calculate_thickness(gtl) gtl = gtl.sel(layer=(th > 0).any(th.dims[1:])) - # add kh and kv to gt - gt = geotop.add_kh_and_kv(gt, geotop_k) - # add kh and kv from gt to gtl gtl = geotop.aggregate_to_ds(gt, gtl) # add gtl-layers to rg-layers - if rg.layer.data[0] == layer: - layer_order = np.concatenate([gtl.layer, rg.layer]) - elif rg.layer.data[-1] == layer: - layer_order = np.concatenate([rg.layer, gtl.layer]) - else: - lay = np.where(rg.layer == layer)[0][0] - layer_order = np.concatenate( - [rg.layer[:lay], gtl.layer, rg.layer[lay + 1 :]] - ) + lay = np.where(rg.layer == layer)[0][0] + layer_order = np.concatenate([rg.layer[:lay], gtl.layer, rg.layer[lay + 1 :]]) + # call xr.concat with rg first, so we keep attributes of rg - rg = xr.concat((rg.sel(layer=rg.layer[rg.layer != layer]), gtl), "layer") + rg = xr.concat((rg, gtl), "layer") # we will then make sure the layer order is right rg = rg.reindex({"layer": layer_order}) return rg @@ -266,13 +277,48 @@ def get_layer_names(): return layer_names -def get_legend(): - """Get a legend (DataFrame) with the colors of REGIS-layers. +def get_legend(kind="REGIS"): + """Get a legend (DataFrame) with the colors of REGIS and/or GeoTOP layers. These colors can be used when plotting cross-sections. """ + allowed_kinds = ["REGIS", "GeoTOP", "combined"] + if kind not in allowed_kinds: + raise (ValueError(f"Only allowed values for kind are {allowed_kinds}")) + if kind in ["REGIS", "combined"]: + dir_path = os.path.dirname(os.path.realpath(__file__)) + fname = os.path.join(dir_path, "..", "data", "regis_2_2.gleg") + leg_regis = read_gleg(fname) + if kind == "REGIS": + return leg_regis + if kind in ["GeoTOP", "combined"]: + dir_path = os.path.dirname(os.path.realpath(__file__)) + fname = os.path.join(dir_path, "..", "data", "geotop", "geotop.gleg") + leg_geotop = read_gleg(fname) + if kind == "GeoTOP": + return leg_geotop + # return a combination of regis and geotop + leg = pd.concat((leg_regis, leg_geotop)) + # drop duplicates, keeping first occurrences (from regis) + leg = leg.loc[~leg.index.duplicated(keep="first")] + return leg + + +def get_legend_lithoclass(): dir_path = os.path.dirname(os.path.realpath(__file__)) - fname = os.path.join(dir_path, "..", "data", "regis_2_2.gleg") + fname = os.path.join(dir_path, "..", "data", "geotop", "Lithoklasse.voleg") + leg = read_voleg(fname) + return leg + + +def get_legend_lithostratigraphy(): + dir_path = os.path.dirname(os.path.realpath(__file__)) + fname = os.path.join(dir_path, "..", "data", "geotop", "Lithostratigrafie.voleg") + leg = read_voleg(fname) + return leg + + +def read_gleg(fname): leg = pd.read_csv( fname, sep="\t", @@ -286,3 +332,18 @@ def get_legend(): leg["color"] = clrs leg = leg.drop(["x", "r", "g", "b", "a"], axis=1) return leg + + +def read_voleg(fname): + leg = pd.read_csv( + fname, + sep="\t", + header=None, + names=["code", "naam", "r", "g", "b", "a", "beschrijving"], + ) + leg.set_index("code", inplace=True) + clrs = np.array(leg.loc[:, ["r", "g", "b"]]) + clrs = [tuple(rgb / 255.0) for rgb in clrs] + leg["color"] = clrs + leg = leg.drop(["r", "g", "b", "a"], axis=1) + return leg diff --git a/nlmod/read/rws.py b/nlmod/read/rws.py index ffe22a6a..7af2a991 100644 --- a/nlmod/read/rws.py +++ b/nlmod/read/rws.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- -"""functions to add surface water to a mf model using the ghb package.""" - import datetime as dt import logging import os import geopandas as gpd +import numpy as np import xarray as xr import nlmod @@ -78,7 +76,7 @@ def get_surface_water(ds, da_basename): peil = xr.where(area_pol > area, row["peil"], peil) area = xr.where(area_pol > area, area_pol, area) - ds_out = util.get_ds_empty(ds) + ds_out = util.get_ds_empty(ds, keep_coords=("y", "x")) ds_out[f"{da_basename}_area"] = area ds_out[f"{da_basename}_area"].attrs["units"] = "m2" ds_out[f"{da_basename}_cond"] = cond @@ -127,7 +125,7 @@ def get_northsea(ds, da_name="northsea"): ) ] - ds_out = dims.gdf_to_bool_ds(swater_zee, ds, da_name) + ds_out = dims.gdf_to_bool_ds(swater_zee, ds, da_name, keep_coords=("y", "x")) return ds_out @@ -153,7 +151,7 @@ def add_northsea(ds, cachedir=None): # fill top, bot, kh, kv at sea cells fal = dims.get_first_active_layer(ds) - fill_mask = (fal == fal.attrs["_FillValue"]) * ds["northsea"] + fill_mask = (fal == fal.attrs["nodata"]) * ds["northsea"] ds = dims.fill_top_bot_kh_kv_at_mask(ds, fill_mask) # add bathymetry noordzee @@ -168,6 +166,105 @@ def add_northsea(ds, cachedir=None): ds = jarkus.add_bathymetry_to_top_bot_kh_kv(ds, ds["bathymetry"], fill_mask) - # update idomain on adjusted tops and bots - ds = dims.set_idomain(ds) + # remove inactive layers + ds = dims.remove_inactive_layers(ds) return ds + + +def calculate_sea_coverage( + dtm, + ds=None, + zmax=0.0, + xy_sea=None, + diagonal=False, + method="mode", + nodata=-1, + return_filled_dtm=False, +): + """ + Determine where the sea is by interpreting the digital terrain model. + + This method assumes the pixel defined in xy_sea (by default top-left) of the + DTM-DataArray is sea. It then determines the height of the sea that is required for + other pixels to become sea as well, taking into account the pixels in between. + + Parameters + ---------- + + dtm : xr.DataArray + The digital terrain data, which can be of higher resolution than ds, Nans are + filled by the minial value of dtm. + ds : xr.Dataset, optional + Dataset with model information. When ds is not None, the sea DataArray is + transformed to the model grid. THe default is None. + zmax : float, optional + Locations thet become sea when the sea level reaches a level of zmax will get a + value of 1 in the resulting DataArray. The default is 0.0. + xy_sea : tuble of 2 floats + The x- and y-coordinate of a location within the dtm that is sea. From this + point, calculate_sea determines at what level each cell becomes wet. When + xy_cell is None, the most northwest grid cell is sea, which is appropriate for + the Netherlands. The default is None. + diagonal : bool, optional + When true, dtm-values are connected diagonally as well (to determine the level + the sea will reach). The default is False. + method : str, optional + The method used to scale the dtm to ds. The default is "mode" (mode means that + if more than half of the (not-nan) cells are wet, the cell is classified as + sea). + nodata : int or float, optional + The value for model cells outside the coverage of the dtm. + Only used internally. The default is -1. + return_filled_dtm : bool, optional + When True, return the filled dtm. The default is False. + + Returns + ------- + sea : xr.DataArray + A DataArray with value of 1 where the sea is and 0 where it is not. + """ + + from skimage.morphology import reconstruction + + if not (dtm < zmax).any(): + logger.warning( + f"There are no values in dtm below {zmax}. The provided dtm " + "probably is not appropriate to calculate the sea boundary." + ) + # fill nans by the minimum value of dtm + dtm = dtm.where(~np.isnan(dtm), dtm.min()) + seed = xr.full_like(dtm, dtm.max()) + if xy_sea is None: + xy_sea = (dtm.x.data.min(), dtm.y.data.max()) + # determine the closest x and y in the dtm grid + x_sea = dtm.x.sel(x=xy_sea[0], method="nearest") + y_sea = dtm.y.sel(y=xy_sea[1], method="nearest") + dtm.loc[{"x": x_sea, "y": y_sea}] = dtm.min() + seed.loc[{"x": x_sea, "y": y_sea}] = dtm.min() + seed = seed.data + + footprint = np.ones((3, 3), dtype="bool") + if not diagonal: + footprint[[0, 0, 2, 2], [0, 2, 2, 0]] = False # no diagonal connections + filled = reconstruction(seed, dtm.data, method="erosion", footprint=footprint) + dtm.data = filled + if return_filled_dtm: + return dtm + + sea_dtm = dtm < zmax + if method == "mode": + sea_dtm = sea_dtm.astype(int) + else: + sea_dtm = sea_dtm.astype(float) + if ds is not None: + sea = nlmod.resample.structured_da_to_ds( + sea_dtm, ds, method=method, nodata=nodata + ) + if (sea == nodata).any(): + logger.info( + "The dtm data does not cover the entire model domain." + " Assuming cells outside dtm-cover to be sea." + ) + sea = sea.where(sea != nodata, 1) + return sea + return sea_dtm diff --git a/nlmod/read/waterboard.py b/nlmod/read/waterboard.py index 8ccdc4e5..3434cb76 100644 --- a/nlmod/read/waterboard.py +++ b/nlmod/read/waterboard.py @@ -8,7 +8,7 @@ def get_polygons(**kwargs): - """Get the location of the Waterboards as a Polygon GeoDataFrame.""" + """Get the location of the Dutch Waterboards as a Polygon GeoDataFrame.""" url = "https://services.arcgis.com/nSZVuSZjHpEZZbRo/arcgis/rest/services/Waterschapsgrenzen/FeatureServer" layer = 0 ws = webservices.arcrest(url, layer, **kwargs) @@ -126,7 +126,7 @@ def get_configuration(): # "layer": 43, # Leggervak droge sloot }, "level_areas": { - "url": "https://geoservices.hdsr.nl/arcgis/rest/services/Extern/PeilbesluitenExtern/FeatureServer", + "url": "https://geoservices.hdsr.nl/arcgis/rest/services/Extern/PeilbesluitenExtern_damo/FeatureServer", "layer": 1, "index": "WS_PGID", "summer_stage": ["WS_ZP", "WS_BP", "WS_OP", "WS_VP"], @@ -522,9 +522,9 @@ def get_data(wb, data_kind, extent=None, max_record_count=None, config=None, **k f = "geojson" if wb not in config: - raise (Exception(f"No configuration available for {wb}")) + raise (ValueError(f"No configuration available for {wb}")) if data_kind not in config[wb]: - raise (Exception(f"{data_kind} not available for {wb}")) + raise (ValueError(f"{data_kind} not available for {wb}")) conf = config[wb][data_kind] url = conf["url"] if "layer" in conf: @@ -551,7 +551,7 @@ def get_data(wb, data_kind, extent=None, max_record_count=None, config=None, **k url, layer, extent, max_record_count=max_record_count, **kwargs ) else: - raise (Exception("Unknown server-kind: {server_kind}")) + raise (ValueError(f"Unknown server-kind: {server_kind}")) if len(gdf) == 0: return gdf diff --git a/nlmod/read/webservices.py b/nlmod/read/webservices.py index 8f000235..17dbd9d3 100644 --- a/nlmod/read/webservices.py +++ b/nlmod/read/webservices.py @@ -10,6 +10,7 @@ from owslib.wcs import WebCoverageService from rasterio import merge from rasterio.io import MemoryFile +from requests.exceptions import HTTPError from shapely.geometry import MultiPolygon, Point, Polygon from tqdm import tqdm @@ -28,7 +29,14 @@ def arcrest( timeout=120, **kwargs, ): - """Download data from an arcgis rest FeatureServer.""" + """Download data from an arcgis rest FeatureServer. + + Note + ---- + The sr argument is left as 28992 and not converted to the EPSG_28992 constant + in the epsg28992.py file in nlmod. Data is probably already picked up in 28992, + and projection issue occurs only when converting to this CRS from another CRS. + """ params = { "f": f, "outFields": "*", @@ -84,7 +92,7 @@ def arcrest( for feature in features: if "rings" in feature["geometry"]: if len(feature["geometry"]) > 1: - raise (Exception("Not supported yet")) + raise (NotImplementedError("Multiple rings not supported yet")) if len(feature["geometry"]["rings"]) == 1: geometry = Polygon(feature["geometry"]["rings"][0]) else: @@ -109,7 +117,11 @@ def arcrest( raise (Exception("Not supported yet")) feature["attributes"]["geometry"] = geometry data.append(feature["attributes"]) - gdf = gpd.GeoDataFrame(data, crs=sr) + if len(data) == 0: + # Assigning CRS to a GeoDataFrame without a geometry column is not supported + gdf = gpd.GeoDataFrame() + else: + gdf = gpd.GeoDataFrame(data, crs=sr) else: # for geojson-data we can transform to GeoDataFrame right away if len(features) == 0: @@ -123,7 +135,7 @@ def arcrest( def _get_data(url, params, timeout=120, **kwargs): r = requests.get(url, params=params, timeout=timeout, **kwargs) if not r.ok: - raise (Exception(f"Request not successful: {r.url}")) + raise (HTTPError(f"Request not successful: {r.url}")) data = r.json() if "error" in data: code = data["error"]["code"] @@ -143,7 +155,7 @@ def wfs( timeout=120, ): """Download data from a wfs server.""" - params = dict(version=version, request="GetFeature") + params = {"version": version, "request": "GetFeature"} if version == "2.0.0": params["typeNames"] = layer else: @@ -155,7 +167,7 @@ def wfs( # get the maximum number of features r = requests.get(f"{url}&request=getcapabilities", timeout=120) if not r.ok: - raise (Exception(f"Request not successful: {r.url}")) + raise (HTTPError(f"Request not successful: {r.url}")) root = ET.fromstring(r.text) ns = {"ows": "http://www.opengis.net/ows/1.1"} @@ -196,7 +208,7 @@ def add_constrains(elem, constraints): params["resultType"] = "hits" r = requests.get(url, params=params, timeout=timeout) if not r.ok: - raise (Exception(f"Request not successful: {r.url}")) + raise (HTTPError(f"Request not successful: {r.url}")) params.pop("resultType") root = ET.fromstring(r.text) if "ExceptionReport" in root.tag: @@ -216,14 +228,14 @@ def add_constrains(elem, constraints): params["startindex"] = ip * max_record_count r = requests.get(url, params=params, timeout=timeout) if not r.ok: - raise (Exception(f"Request not successful: {r.url}")) + raise (HTTPError(f"Request not successful: {r.url}")) gdfs.append(gpd.read_file(BytesIO(r.content), driver=driver)) gdf = pd.concat(gdfs).reset_index(drop=True) else: # download all features in one go r = requests.get(url, params=params, timeout=timeout) if not r.ok: - raise (Exception(f"Request not successful: {r.url}")) + raise (HTTPError(f"Request not successful: {r.url}")) gdf = gpd.read_file(BytesIO(r.content), driver=driver) return gdf @@ -422,7 +434,7 @@ def _download_wcs(extent, res, url, identifier, version, fmt, crs): if identifier is None: identifiers = list(wcs.contents) if len(identifiers) > 1: - raise (Exception("wcs contains more than 1 identifier. Please specify.")) + raise (ValueError("wcs contains more than 1 identifier. Please specify.")) identifier = identifiers[0] if version == "1.0.0": bbox = (extent[0], extent[2], extent[1], extent[3]) @@ -441,9 +453,9 @@ def _download_wcs(extent, res, url, identifier, version, fmt, crs): identifier=[identifier], subsets=subsets, format=fmt, crs=crs ) else: - raise Exception(f"Version {version} not yet supported") + raise NotImplementedError(f"Version {version} not yet supported") if "xml" in output.info()["Content-Type"]: root = ET.fromstring(output.read()) - raise (Exception("Download failed: {}".format(root[0].text))) + raise (HTTPError(f"Download failed: {root[0].text}")) memfile = MemoryFile(output.read()) return memfile diff --git a/nlmod/sim/sim.py b/nlmod/sim/sim.py index e31a2ec9..119a97bf 100644 --- a/nlmod/sim/sim.py +++ b/nlmod/sim/sim.py @@ -1,8 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Thu Jan 7 17:20:34 2021. - -@author: oebbe -""" import datetime as dt import logging import os @@ -17,14 +12,10 @@ logger = logging.getLogger(__name__) -def write_and_run(sim, ds, write_ds=True, nb_path=None, silent=False): - """write modflow files and run the model. - - 2 extra options: - 1. write the model dataset to cache - 2. copy the modelscript (typically a Jupyter Notebook) to the model - workspace with a timestamp. - +def write_and_run(sim, ds, write_ds=True, script_path=None, silent=False): + """write modflow files and run the model. Extra options include writing the + model dataset to a netcdf file in the model workspace and copying the + modelscript to the model workspace. Parameters ---------- @@ -33,33 +24,34 @@ def write_and_run(sim, ds, write_ds=True, nb_path=None, silent=False): ds : xarray.Dataset dataset with model data. write_ds : bool, optional - if True the model dataset is cached to a NetCDF-file (.nc) with a name equal - to its attribute called "model_name". The default is True. - nb_path : str or None, optional - full path of the Jupyter Notebook (.ipynb) with the modelscript. The - default is None. Preferably this path does not have to be given - manually but there is currently no good option to obtain the filename - of a Jupyter Notebook from within the notebook itself. + if True the model dataset is written to a NetCDF-file (.nc) in the + model workspace the name of the .nc file is used from the attribute + "model_name". The default is True. + script_path : str or None, optional + full path of the Jupyter Notebook (.ipynb) or the module (.py) with the + modelscript. The default is None. Preferably this path does not have to + be given manually but there is currently no good option to obtain the + filename of a Jupyter Notebook from within the notebook itself. silent : bool, optional write and run model silently """ if isinstance(sim, flopy.mf6.ModflowGwf): sim = sim.simulation - if nb_path is not None: - new_nb_fname = ( - f'{dt.datetime.now().strftime("%Y%m%d")}' + os.path.split(nb_path)[-1] + if script_path is not None: + new_script_fname = ( + f'{dt.datetime.now().strftime("%Y%m%d")}' + os.path.split(script_path)[-1] ) - dst = os.path.join(ds.model_ws, new_nb_fname) - logger.info(f"write script {new_nb_fname} to model workspace") - copyfile(nb_path, dst) + dst = os.path.join(ds.model_ws, new_script_fname) + logger.info(f"write script {new_script_fname} to model workspace") + copyfile(script_path, dst) if write_ds: logger.info("write model dataset to cache") ds.attrs["model_dataset_written_to_disk_on"] = dt.datetime.now().strftime( "%Y%m%d_%H:%M:%S" ) - ds.to_netcdf(os.path.join(ds.attrs["cachedir"], f"{ds.model_name}.nc")) + ds.to_netcdf(os.path.join(ds.attrs["model_ws"], f"{ds.model_name}.nc")) logger.info("write modflow files to model workspace") sim.write_simulation(silent=silent) @@ -72,7 +64,7 @@ def write_and_run(sim, ds, write_ds=True, nb_path=None, silent=False): ds.attrs["model_ran_on"] = dt.datetime.now().strftime("%Y%m%d_%H:%M:%S") -def get_tdis_perioddata(ds): +def get_tdis_perioddata(ds, nstp="nstp", tsmult="tsmult"): """Get tdis_perioddata from ds. Parameters @@ -100,15 +92,15 @@ def get_tdis_perioddata(ds): if len(ds["time"]) > 1: perlen.extend(np.diff(ds["time"]) / deltat) - if "nstp" in ds: - nstp = ds["nstp"].values - else: - nstp = [ds.time.nstp] * len(perlen) + nstp = util._get_value_from_ds_datavar(ds, "nstp", nstp, return_da=False) + + if isinstance(nstp, (int, np.integer)): + nstp = [nstp] * len(perlen) - if "tsmult" in ds: - tsmult = ds["tsmult"].values - else: - tsmult = [ds.time.tsmult] * len(perlen) + tsmult = util._get_value_from_ds_datavar(ds, "tsmult", tsmult, return_da=False) + + if isinstance(tsmult, float): + tsmult = [tsmult] * len(perlen) tdis_perioddata = list(zip(perlen, nstp, tsmult)) @@ -136,7 +128,7 @@ def sim(ds, exe_name=None): """ # start creating model - logger.info("creating modflow SIM") + logger.info("creating mf6 SIM") if exe_name is None: exe_name = util.get_exe_path(ds.mfversion) @@ -152,7 +144,7 @@ def sim(ds, exe_name=None): return sim -def tdis(ds, sim, pname="tdis"): +def tdis(ds, sim, pname="tdis", nstp="nstp", tsmult="tsmult", **kwargs): """create tdis package from the model dataset. Parameters @@ -164,6 +156,8 @@ def tdis(ds, sim, pname="tdis"): simulation object. pname : str, optional package name + **kwargs + passed on to flopy.mft.ModflowTdis Returns ------- @@ -172,18 +166,19 @@ def tdis(ds, sim, pname="tdis"): """ # start creating model - logger.info("creating modflow TDIS") + logger.info("creating mf6 TDIS") - tdis_perioddata = get_tdis_perioddata(ds) + tdis_perioddata = get_tdis_perioddata(ds, nstp=nstp, tsmult=tsmult) # Create the Flopy temporal discretization object - tdis = flopy.mf6.modflow.mftdis.ModflowTdis( + tdis = flopy.mf6.ModflowTdis( sim, pname=pname, time_units=ds.time.time_units, nper=len(ds.time), start_date_time=pd.Timestamp(ds.time.start).isoformat(), perioddata=tdis_perioddata, + **kwargs, ) return tdis @@ -207,13 +202,15 @@ def ims(sim, complexity="MODERATE", pname="ims", **kwargs): ims object. """ - logger.info("creating modflow IMS") + logger.info("creating mf6 IMS") + + print_option = kwargs.pop("print_option", "summary") # Create the Flopy iterative model solver (ims) Package object ims = flopy.mf6.ModflowIms( sim, pname=pname, - print_option="summary", + print_option=print_option, complexity=complexity, **kwargs, ) diff --git a/nlmod/util.py b/nlmod/util.py index 9189f329..4abf90b8 100644 --- a/nlmod/util.py +++ b/nlmod/util.py @@ -15,10 +15,50 @@ logger = logging.getLogger(__name__) +class LayerError(Exception): + """Generic error when modifying layers.""" + + +class MissingValueError(Exception): + """Generic error when an expected value is not defined.""" + + +def check_da_dims_coords(da, ds): + """Check if DataArray dimensions and coordinates match those in Dataset. + + Only checks overlapping dimensions. + + Parameters + ---------- + da : xarray.DataArray + dataarray to check + ds : xarray.Dataset or xarray.DataArray + dataset or dataarray to compare coords with + + Returns + ------- + True + if dimensions and coordinates match + + Raises + ------ + AssertionError + error that coordinates do not match + """ + shared_dims = set(da.dims) & set(ds.dims) + for dim in shared_dims: + try: + xr.testing.assert_identical(da[dim], ds[dim]) + except AssertionError as e: + logger.error(f"da '{da.name}' coordinates do not match ds!") + raise e + return True + + def get_model_dirs(model_ws): - """Creates a new model workspace directory, if it does not exists yet. - Within the model workspace directory a few subdirectories are created (if - they don't exist yet): + """Creates a new model workspace directory, if it does not exists yet. Within the + model workspace directory a few subdirectories are created (if they don't exist + yet): - figure - cache @@ -50,8 +90,7 @@ def get_model_dirs(model_ws): def get_exe_path(exe_name="mf6"): - """get the full path of the executable. Uses the bin directory in the nlmod - package. + """Get the full path of the executable. Uses the bin directory in the nlmod package. Parameters ---------- @@ -75,29 +114,35 @@ def get_exe_path(exe_name="mf6"): return exe_path -def get_ds_empty(ds): - """get a copy of a model dataset with only coordinate information. +def get_ds_empty(ds, keep_coords=None): + """Get a copy of a dataset with only coordinate information. Parameters ---------- ds : xr.Dataset dataset with coordinates + keep_coords : tuple or None, optional + the coordinates in ds the you want keep in your empty ds. If None all + coordinates are kept from original ds. The default is None. Returns ------- empty_ds : xr.Dataset - dataset with only model coordinate information + dataset with only coordinate information """ + if keep_coords is None: + keep_coords = list(ds.coords) empty_ds = xr.Dataset() for coord in list(ds.coords): - empty_ds = empty_ds.assign_coords(coords={coord: ds[coord]}) + if coord in keep_coords: + empty_ds = empty_ds.assign_coords(coords={coord: ds[coord]}) return empty_ds def get_da_from_da_ds(da_ds, dims=("y", "x"), data=None): - """get a dataarray from ds with certain dimensions. + """Get a dataarray from ds with certain dimensions. Parameters ---------- @@ -130,9 +175,8 @@ def get_da_from_da_ds(da_ds, dims=("y", "x"), data=None): def find_most_recent_file(folder, name, extension=".pklz"): - """find the most recent file in a folder. File must startwith name and - end width extension. If you want to look for the most recent folder use - extension = ''. + """Find the most recent file in a folder. File must startwith name and end width + extension. If you want to look for the most recent folder use extension = ''. Parameters ---------- @@ -169,7 +213,7 @@ def find_most_recent_file(folder, name, extension=".pklz"): def compare_model_extents(extent1, extent2): - """check overlap between two model extents. + """Check overlap between two model extents. Parameters ---------- @@ -224,7 +268,7 @@ def compare_model_extents(extent1, extent2): def polygon_from_extent(extent): - """create a shapely polygon from a given extent. + """Create a shapely polygon from a given extent. Parameters ---------- @@ -244,7 +288,7 @@ def polygon_from_extent(extent): def gdf_from_extent(extent, crs="EPSG:28992"): - """create a geodataframe with a single polygon with the extent given. + """Create a geodataframe with a single polygon with the extent given. Parameters ---------- @@ -267,8 +311,8 @@ def gdf_from_extent(extent, crs="EPSG:28992"): def gdf_within_extent(gdf, extent): - """select only parts of the geodataframe within the extent. Only accepts - Polygon and Linestring geometry types. + """Select only parts of the geodataframe within the extent. Only accepts Polygon and + Linestring geometry types. Parameters ---------- @@ -307,7 +351,7 @@ def gdf_within_extent(gdf, extent): def get_google_drive_filename(fid, timeout=120): - """get the filename of a google drive file. + """Get the filename of a google drive file. Parameters ---------- @@ -335,7 +379,7 @@ def get_google_drive_filename(fid, timeout=120): def download_file_from_google_drive(fid, destination=None): - """download a file from google drive using it's id. + """Download a file from google drive using it's id. Parameters ---------- @@ -411,14 +455,24 @@ def download_mfbinaries(bindir=None): def download_modpath_provisional_exe(bindir=None, timeout=120): - """Downlaod the provisional version of modpath to the folder with binaries""" + """Download the provisional version of modpath to the folder with binaries.""" if bindir is None: bindir = os.path.join(os.path.dirname(__file__), "bin") if not os.path.isdir(bindir): os.makedirs(bindir) - url = "https://github.com/MODFLOW-USGS/modpath-v7/raw/develop/msvs/bin_PROVISIONAL/mpath7_PROVISIONAL_2022-08-23_9ac760f.exe" + if sys.platform.startswith("win"): + fname = "mp7_win64_20230911_8eca8d8.exe" + elif sys.platform.startswith("darwin"): + fname = "mp7_mac_20230911_8eca8d8" + elif sys.platform.startswith("linux"): + fname = "mp7_linux_20230911_8eca8d8" + else: + raise (OSError(f"Unknown platform: {sys.platform}")) + url = "https://github.com/MODFLOW-USGS/modpath-v7/raw/develop/msvs/bin_PROVISIONAL" + url = f"{url}/{fname}" r = requests.get(url, allow_redirects=True, timeout=timeout) - fname = os.path.join(bindir, "mp7_2_002_provisional.exe") + ext = os.path.splitext(fname)[-1] + fname = os.path.join(bindir, f"mp7_2_002_provisional{ext}") with open(fname, "wb") as file: file.write(r.content) @@ -468,8 +522,12 @@ def format(self, record) -> str: def get_color_logger(level="INFO"): + if level == "DEBUG": + FORMAT = "{color}{levelname}:{name}.{funcName}:{lineno}:{message}{reset}" + else: + FORMAT = "{color}{levelname}:{name}.{funcName}:{message}{reset}" formatter = ColoredFormatter( - "{color}{levelname}:{name}:{message}{reset}", + FORMAT, style="{", datefmt="%Y-%m-%d %H:%M:%S", colors={ @@ -493,7 +551,9 @@ def get_color_logger(level="INFO"): return logger -def _get_value_from_ds_attr(ds, varname, attr=None, value=None, warn=True): +def _get_value_from_ds_attr( + ds, varname, attr=None, value=None, default=None, warn=True +): """Internal function to get value from dataset attributes. Parameters @@ -506,6 +566,9 @@ def _get_value_from_ds_attr(ds, varname, attr=None, value=None, warn=True): name of the attribute in dataset (is sometimes different to varname) value : Any, optional variable value, by default None + default : Any, optional + When default is not None, value is None, and attr is not present in ds, + this default is returned. The default is None. warn : bool, optional log warning if value not found @@ -526,7 +589,10 @@ def _get_value_from_ds_attr(ds, varname, attr=None, value=None, warn=True): logger.debug(f"Using stored data attribute '{attr}' for '{varname}'") value = ds.attrs[attr] elif value is None: - if warn: + if default is not None: + logger.debug(f"Using default value of {default} for '{varname}'") + value = default + elif warn: msg = ( f"No value found for '{varname}', returning None. " f"To fix this error pass '{varname}' to function or set 'ds.{attr}'." @@ -536,7 +602,15 @@ def _get_value_from_ds_attr(ds, varname, attr=None, value=None, warn=True): return value -def _get_value_from_ds_datavar(ds, varname, datavar=None, warn=True): +def _get_value_from_ds_datavar( + ds, + varname, + datavar=None, + default=None, + warn=True, + return_da=False, + checkcoords=True, +): """Internal function to get value from dataset data variables. Parameters @@ -544,13 +618,22 @@ def _get_value_from_ds_datavar(ds, varname, datavar=None, warn=True): ds : xarray.Dataset dataset containing model data varname : str - name of the variable in flopy package + name of the variable in flopy package (used for logging) datavar : Any, optional if str, treated as the name of the data variable (which can be different to varname) in dataset, if not provided is assumed to be the same as varname. If not passed as string, it is treated as data + default : Any, optional + When default is not None, datavar is a string, and not present in ds, this + default is returned. The default is None. warn : bool, optional log warning if value not found + return_da : bool, optional + if True, a DataArray can be returned. If False, a DataArray is always + converted to a numpy array before being returned. The default is False. + checkcoords : bool, optional + if True and datavar is a DataArray, the DataArray coords are checked against + the Dataset coordinates. Raises an AssertionError if they do not match. Returns ------- @@ -571,6 +654,8 @@ def _get_value_from_ds_datavar(ds, varname, datavar=None, warn=True): if isinstance(datavar, xr.DataArray): value = datavar datavar = datavar.name + if checkcoords: + check_da_dims_coords(value, ds) elif isinstance(datavar, str): value = datavar else: @@ -589,12 +674,20 @@ def _get_value_from_ds_datavar(ds, varname, datavar=None, warn=True): value = ds[datavar] # warn if value is None elif isinstance(value, str): - value = None - if warn: - msg = ( - f"No value found for '{varname}', returning None. " - f"To silence this warning pass '{varname}' data directly " - f"to function or check whether 'ds.{datavar}' was set correctly." - ) - logger.warning(msg) + if default is not None: + logger.debug(f"Using default value of {default} for '{varname}'") + value = default + else: + value = None + if warn: + msg = ( + f"No value found for '{varname}', returning None. " + f"To silence this warning pass '{varname}' data directly " + f"to function or check whether 'ds.{datavar}' was set correctly." + ) + logger.warning(msg) + if not return_da: + if isinstance(value, xr.DataArray): + value = value.values + return value diff --git a/nlmod/version.py b/nlmod/version.py index 86efe7c1..9bae5790 100644 --- a/nlmod/version.py +++ b/nlmod/version.py @@ -1,7 +1,7 @@ from importlib import metadata from platform import python_version -__version__ = "0.6.1" +__version__ = "0.7.0" def show_versions() -> None: diff --git a/pyproject.toml b/pyproject.toml index 7f4306a5..d63b685c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "nlmod" dynamic = ["version"] -description = "nlmod is an open-source Python package for building Modflow 6 groundwater models from online data sources in The Netherlands" +description = "Python package to build, run and visualize MODFLOW 6 groundwater models in the Netherlands." license = { file = "LICENSE" } readme = "README.md" authors = [ @@ -23,13 +23,13 @@ requires-python = ">= 3.8" dependencies = [ "flopy>=3.3.6", "xarray>=0.16.1", - "netcdf4>=1.5.7", + "netcdf4>=1.6.3", "rasterio>=1.1.0", "rioxarray", "affine>=0.3.1", "geopandas", "owslib>=0.24.1", - "hydropandas>=0.7.1", + "hydropandas>=0.9.2", "shapely>=2.0.0", "pyshp>=2.1.3", "matplotlib", @@ -55,13 +55,20 @@ repository = "https://github.com/ArtesiaWater/nlmod" documentation = "https://nlmod.readthedocs.io/en/latest/" [project.optional-dependencies] -full = ["nlmod[knmi]", "gdown", "geocube", "bottleneck", "contextily"] +full = [ + "nlmod[knmi]", + "gdown", + "geocube", + "bottleneck", + "contextily", + "scikit-image", +] knmi = ["h5netcdf", "nlmod[grib]"] grib = ["cfgrib", "ecmwflibs"] test = ["pytest>=7", "pytest-cov", "pytest-dependency"] nbtest = ["nbformat", "nbconvert>6.4.5"] lint = ["flake8", "isort", "black[jupyter]"] -ci = ["nlmod[full,lint,test,nbtest]", "netCDF4==1.5.7"] +ci = ["nlmod[full,lint,test,nbtest]", "netCDF4>=1.6.3", "pandas<2.1.0"] rtd = [ "nlmod[full]", "ipython", @@ -69,7 +76,7 @@ rtd = [ "nbsphinx", "sphinx_rtd_theme==1.0.0", "nbconvert>6.4.5", - "netCDF4==1.5.7", + "netCDF4>=1.6.3", ] [tool.setuptools.dynamic] diff --git a/tests/data/mf6output/structured/test.cbc b/tests/data/mf6output/structured/test.cbc new file mode 100644 index 00000000..a87abdbc Binary files /dev/null and b/tests/data/mf6output/structured/test.cbc differ diff --git a/tests/data/mf6output/structured/test.dis.grb b/tests/data/mf6output/structured/test.dis.grb new file mode 100644 index 00000000..8c194968 Binary files /dev/null and b/tests/data/mf6output/structured/test.dis.grb differ diff --git a/tests/data/mf6output/structured/test.hds b/tests/data/mf6output/structured/test.hds new file mode 100644 index 00000000..09f051ed Binary files /dev/null and b/tests/data/mf6output/structured/test.hds differ diff --git a/tests/data/mf6output/vertex/test.cbc b/tests/data/mf6output/vertex/test.cbc new file mode 100644 index 00000000..9b013f10 Binary files /dev/null and b/tests/data/mf6output/vertex/test.cbc differ diff --git a/tests/data/mf6output/vertex/test.disv.grb b/tests/data/mf6output/vertex/test.disv.grb new file mode 100644 index 00000000..510b51ce Binary files /dev/null and b/tests/data/mf6output/vertex/test.disv.grb differ diff --git a/tests/data/mf6output/vertex/test.hds b/tests/data/mf6output/vertex/test.hds new file mode 100644 index 00000000..caaacdbf Binary files /dev/null and b/tests/data/mf6output/vertex/test.hds differ diff --git a/tests/test_001_model.py b/tests/test_001_model.py index 9c9c1362..d73130a0 100644 --- a/tests/test_001_model.py +++ b/tests/test_001_model.py @@ -1,6 +1,8 @@ import os import tempfile +import numpy as np +import pandas as pd import pytest import xarray as xr @@ -15,23 +17,30 @@ def test_model_directories(tmpdir): figdir, cachedir = nlmod.util.get_model_dirs(model_ws) +def test_snap_extent(): + extent = (0.22, 1056.12, 7.43, 1101.567) + new_extent = nlmod.dims.snap_extent(extent, 10, 20) + assert new_extent == [0.0, 1060.0, 0.0, 1120.0] + + extent = (1000, 2000, 8000, 10000) + new_extent = nlmod.dims.snap_extent(extent, 250, 55) + assert new_extent == [1000.0, 2000.0, 7975.0, 10010.0] + + def get_ds_time_steady(tmpdir, modelname="test"): model_ws = os.path.join(tmpdir, "test_model") ds = nlmod.base.set_ds_attrs(xr.Dataset(), modelname, model_ws) - ds = nlmod.time.set_ds_time(ds, start_time="2015-1-1", steady_state=True) + ds = nlmod.time.set_ds_time(ds, time=["2015-1-2"], start="2015-1-1", steady=True) return ds def get_ds_time_transient(tmpdir, modelname="test"): model_ws = os.path.join(tmpdir, "test_model") ds = nlmod.base.set_ds_attrs(xr.Dataset(), modelname, model_ws) - ds = nlmod.time.set_ds_time( - ds, - start_time="2015-1-1", - steady_state=False, - steady_start=True, - transient_timesteps=10, - ) + nper = 11 + time = pd.date_range(start="2015-1-2", periods=nper, freq="D") + steady = np.zeros(nper) + ds = nlmod.time.set_ds_time(ds, time=time, start="2015-1-1", steady=steady) return ds @@ -79,12 +88,14 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): ) assert ds.dims["layer"] == 5 + nper = 11 + steady = np.zeros(nper, dtype=int) + steady[0] = 1 ds = nlmod.time.set_ds_time( ds, - start_time="2015-1-1", - steady_state=False, - steady_start=True, - transient_timesteps=10, + time=pd.date_range("2015-1-2", periods=nper, freq="D"), + start="2015-1-1", + steady=steady, ) # create simulation @@ -118,12 +129,14 @@ def test_create_sea_model_grid_only(tmpdir, model_name="test"): regis_geotop_ds, model_name, model_ws, delr=100.0, delc=100.0 ) + nper = 11 + steady = np.zeros(nper, dtype=int) + steady[0] = 1 ds = nlmod.time.set_ds_time( ds, - start_time="2015-1-1", - steady_state=False, - steady_start=True, - transient_timesteps=10, + time=pd.date_range("2015-1-2", periods=nper, freq="D"), + start="2005-1-1", + steady=steady, ) # save ds @@ -186,7 +199,7 @@ def test_create_sea_model(tmpdir): _ = nlmod.gwf.surface_drain_from_ds(ds, gwf, 0.1) # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds, ds["idomain"])) + ds.update(nlmod.grid.mask_model_edge(ds)) _ = nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") # add knmi recharge to the model datasets @@ -206,9 +219,10 @@ def test_create_sea_model_perlen_list(tmpdir): ds = nlmod.base.set_ds_attrs(ds, ds.model_name, model_ws) # create transient with perlen list + start = ds.time.start perlen = [3650, 14, 10, 11] # length of the time steps - transient_timesteps = 3 - start_time = ds.time.start + steady = np.zeros(len(perlen), dtype=int) + steady[0] = 1 # drop time dimension before setting time ds = ds.drop_dims("time") @@ -216,11 +230,9 @@ def test_create_sea_model_perlen_list(tmpdir): # update current ds with new time dicretisation ds = nlmod.time.set_ds_time( ds, - start_time=start_time, - steady_state=False, - steady_start=True, - perlen=perlen, - transient_timesteps=transient_timesteps, + time=np.cumsum(perlen), + start=start, + steady=steady, ) # create simulation @@ -257,7 +269,7 @@ def test_create_sea_model_perlen_list(tmpdir): nlmod.gwf.surface_drain_from_ds(ds, gwf, 1.0) # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds, ds["idomain"])) + ds.update(nlmod.grid.mask_model_edge(ds)) nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") # add knmi recharge to the model datasets @@ -277,21 +289,24 @@ def test_create_sea_model_perlen_14(tmpdir): ds = nlmod.base.set_ds_attrs(ds, ds.model_name, model_ws) # create transient with perlen list - perlen = 14 # length of the time steps - transient_timesteps = 3 - start_time = ds.time.start + perlen = 14 # length of the transient time steps + nper = 4 + start = ds.time.start + perlen = perlen * np.ones(nper) + perlen[0] = 3652.0 # length of the steady state step + steady = np.zeros(nper, dtype=int) + steady[0] = 1 + time = nlmod.time.ds_time_idx_from_tdis_settings(start, perlen=perlen) # drop time dimension before setting time ds = ds.drop_dims("time") - # update current ds with new time dicretisation + # update current ds with new time discretization ds = nlmod.time.set_ds_time( ds, - start_time=start_time, - steady_state=False, - steady_start=True, - perlen=perlen, - transient_timesteps=transient_timesteps, + time=time, + start=start, + steady=steady, ) # create simulation @@ -328,7 +343,7 @@ def test_create_sea_model_perlen_14(tmpdir): nlmod.gwf.surface_drain_from_ds(ds, gwf, 1.0) # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds, ds["idomain"])) + ds.update(nlmod.grid.mask_model_edge(ds)) nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") # add knmi recharge to the model datasets diff --git a/tests/test_002_regis_geotop.py b/tests/test_002_regis_geotop.py index 3b4ec026..423593bb 100644 --- a/tests/test_002_regis_geotop.py +++ b/tests/test_002_regis_geotop.py @@ -1,9 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Thu Jan 7 16:23:35 2021. - -@author: oebbe -""" - import nlmod @@ -24,18 +18,13 @@ def test_get_regis_botm_layer_BEk1( assert regis_ds.layer.values[-1] == botm_layer -def test_get_geotop_raw(extent=[98600.0, 99000.0, 489400.0, 489700.0]): - geotop_ds = nlmod.read.geotop.get_geotop_raw_within_extent(extent) +def test_get_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): + geotop_ds = nlmod.read.geotop.get_geotop(extent) line = [(extent[0], extent[2]), (extent[1], extent[3])] # also test the plot-method nlmod.plot.geotop_lithok_in_cross_section(line, geotop_ds) -# @pytest.mark.skip(reason="too slow") -def test_get_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): - nlmod.read.geotop.get_geotop(extent) - - # @pytest.mark.skip(reason="too slow") def test_get_regis_geotop(extent=[98600.0, 99000.0, 489400.0, 489700.0]): regis_geotop_ds = nlmod.read.regis.get_combined_layer_models( diff --git a/tests/test_003_mfpackages.py b/tests/test_003_mfpackages.py index 89654be0..6468de9c 100644 --- a/tests/test_003_mfpackages.py +++ b/tests/test_003_mfpackages.py @@ -1,8 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Thu Jan 7 16:24:03 2021. - -@author: oebbe -""" import numpy as np import test_001_model import xarray as xr @@ -86,7 +81,7 @@ def chd_from_ds(tmpdir): _ = nlmod.gwf.ic(ds, gwf, starting_head=1.0) # add constant head cells at model boundaries - ds.update(nlmod.grid.mask_model_edge(ds, ds["idomain"])) + ds.update(nlmod.grid.mask_model_edge(ds)) nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") @@ -102,20 +97,22 @@ def get_value_from_ds_datavar(): ds["test_var"] = ("layer", "y", "x"), np.arange(np.product(shape)).reshape(shape) # get value from ds - v0 = nlmod.util._get_value_from_ds_datavar(ds, "test_var", "test_var") + v0 = nlmod.util._get_value_from_ds_datavar( + ds, "test_var", "test_var", return_da=True + ) xr.testing.assert_equal(ds["test_var"], v0) # get value from ds, variable and stored name are different v1 = nlmod.util._get_value_from_ds_datavar(ds, "test", "test_var") - xr.testing.assert_equal(ds["test_var"], v1) + xr.testing.assert_equal(ds["test_var"].values, v1) # do not get value from ds, value is Data Array, should log info msg - v2 = nlmod.util._get_value_from_ds_datavar(ds, "test", v0) + v2 = nlmod.util._get_value_from_ds_datavar(ds, "test", v0, return_da=True) xr.testing.assert_equal(ds["test_var"], v2) # do not get value from ds, value is Data Array, no msg v0.name = "test2" - v3 = nlmod.util._get_value_from_ds_datavar(ds, "test", v0) + v3 = nlmod.util._get_value_from_ds_datavar(ds, "test", v0, return_da=True) assert (v0 == v3).all() # return None, value is str but not in dataset, should log warning diff --git a/tests/test_004_northsea.py b/tests/test_004_northsea.py index ea6c3ef0..2e49171d 100644 --- a/tests/test_004_northsea.py +++ b/tests/test_004_northsea.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import test_001_model import nlmod @@ -39,7 +37,7 @@ def test_fill_top_bot_kh_kv_seamodel(): ds.update(nlmod.read.rws.get_northsea(ds)) fal = nlmod.layers.get_first_active_layer(ds) - fill_mask = (fal == fal._FillValue) * ds["northsea"] + fill_mask = (fal == fal.nodata) * ds["northsea"] nlmod.layers.fill_top_bot_kh_kv_at_mask(ds, fill_mask) @@ -49,7 +47,7 @@ def test_fill_top_bot_kh_kv_nosea(): ds.update(nlmod.read.rws.get_northsea(ds)) fal = nlmod.layers.get_first_active_layer(ds) - fill_mask = (fal == fal._FillValue) * ds["northsea"] + fill_mask = (fal == fal.nodata) * ds["northsea"] nlmod.layers.fill_top_bot_kh_kv_at_mask(ds, fill_mask) @@ -78,6 +76,6 @@ def test_add_bathymetrie_to_top_bot_kh_kv_seamodel(): ds.update(nlmod.read.jarkus.get_bathymetry(ds, ds["northsea"])) fal = nlmod.layers.get_first_active_layer(ds) - fill_mask = (fal == fal._FillValue) * ds["northsea"] + fill_mask = (fal == fal.nodata) * ds["northsea"] nlmod.read.jarkus.add_bathymetry_to_top_bot_kh_kv(ds, ds["bathymetry"], fill_mask) diff --git a/tests/test_005_external_data.py b/tests/test_005_external_data.py index 42d32323..da72fe7a 100644 --- a/tests/test_005_external_data.py +++ b/tests/test_005_external_data.py @@ -1,3 +1,5 @@ +import pandas as pd +import pytest import test_001_model import nlmod @@ -11,9 +13,10 @@ def test_get_recharge(): ds.update(nlmod.read.knmi.get_recharge(ds)) -def test_get_reacharge_most_common(): +def test_get_recharge_most_common(): # model with sea - ds = test_001_model.get_ds_from_cache("basic_sea_model") + ds = nlmod.get_ds([100000, 110000, 420000, 430000]) + ds = nlmod.time.set_ds_time(ds, start="2021", time=pd.date_range("2022", "2023")) # add knmi recharge to the model dataset ds.update(nlmod.read.knmi.get_recharge(ds, most_common_station=True)) @@ -25,12 +28,20 @@ def test_get_recharge_steady_state(): # modify mtime ds = ds.drop_dims("time") - ds = nlmod.time.set_ds_time(ds, start_time="2000-1-1", perlen=3650) + ds = nlmod.time.set_ds_time(ds, time=[3650], start="2000-1-1") # add knmi recharge to the model dataset ds.update(nlmod.read.knmi.get_recharge(ds)) +def test_get_recharge_not_available(): + ds = nlmod.get_ds([100000, 110000, 420000, 430000]) + time = pd.date_range("2010", pd.Timestamp.now()) + ds = nlmod.time.set_ds_time(ds, start="2000", time=time) + with pytest.raises(ValueError): + ds.update(nlmod.read.knmi.get_recharge(ds)) + + def test_ahn_within_extent(): extent = [95000.0, 105000.0, 494000.0, 500000.0] da = nlmod.read.ahn.get_ahn_from_wcs(extent) diff --git a/tests/test_006_caching.py b/tests/test_006_caching.py index 902445a0..741c1ffd 100644 --- a/tests/test_006_caching.py +++ b/tests/test_006_caching.py @@ -1,9 +1,3 @@ -# -*- coding: utf-8 -*- -"""Created on Mon Jan 11 12:26:16 2021. - -@author: oebbe -""" - import tempfile import pytest @@ -46,6 +40,15 @@ def test_ds_check_time_attributes_false(): assert not check +def test_cache_data_array(): + extent = [119_900, 120_000, 441_900, 442_000] + ahn_no_cache = nlmod.read.ahn.get_ahn4(extent) + ahn_cached = nlmod.read.ahn.get_ahn4(extent, cachedir=tmpdir, cachename="ahn4.nc") + ahn_cache = nlmod.read.ahn.get_ahn4(extent, cachedir=tmpdir, cachename="ahn4.nc") + assert ahn_cached.equals(ahn_no_cache) + assert ahn_cache.equals(ahn_no_cache) + + @pytest.mark.slow def test_ds_check_grid_false(tmpdir): # two models with a different grid and same time dicretisation diff --git a/tests/test_007_run_notebooks.py b/tests/test_007_run_notebooks.py index 985ea714..1ed78c4d 100644 --- a/tests/test_007_run_notebooks.py +++ b/tests/test_007_run_notebooks.py @@ -102,3 +102,8 @@ def test_run_notebook_15_geotop(): @pytest.mark.notebooks def test_run_notebook_16_groundwater_transport(): _run_notebook(nbdir, "16_groundwater_transport.ipynb") + + +@pytest.mark.notebooks +def test_run_notebook_17_unsaturated_zone_flow(): + _run_notebook(nbdir, "17_unsaturated_zone_flow.ipynb") diff --git a/tests/test_008_waterschappen.py b/tests/test_008_waterschappen.py index 0ec2b10b..bd9a6a0b 100644 --- a/tests/test_008_waterschappen.py +++ b/tests/test_008_waterschappen.py @@ -1,11 +1,6 @@ -# -*- coding: utf-8 -*- -"""Created on Tue Aug 16 10:29:13 2022. - -@author: Ruben -""" - -import pytest import matplotlib +import pytest + import nlmod diff --git a/tests/test_009_layers.py b/tests/test_009_layers.py index 3774b9ec..ac1d44b6 100644 --- a/tests/test_009_layers.py +++ b/tests/test_009_layers.py @@ -1,6 +1,7 @@ import os import matplotlib.pyplot as plt +import numpy as np from shapely.geometry import LineString import nlmod @@ -104,3 +105,24 @@ def test_set_layer_botm(plot=False): if plot: plot_test(ds, ds_new) + + +def test_insert_layer(): + ds1 = get_regis_horstermeer() + # just replace the 2nd layer by a new insertion + layer = ds1.layer.data[1] + new_layer = "test" + ds2 = nlmod.layers.insert_layer( + ds1, "test", ds1["top"].loc[layer], ds1["botm"].loc[layer] + ) + # total_thickness1 = float(nlmod.layers.calculate_thickness(ds1).sum()) + # total_thickness2 = float(nlmod.layers.calculate_thickness(ds2).sum()) + # assert total_thickness1 == total_thickness2 + assert nlmod.layers.calculate_thickness(ds2).loc[layer].sum() == 0 + mask = ~np.isnan(ds1["top"].loc[layer]) + assert ( + ds2["top"].loc[new_layer].data[mask] == ds1["top"].loc[layer].data[mask] + ).all() + assert ( + ds2["botm"].loc[new_layer].data[mask] == ds1["botm"].loc[layer].data[mask] + ).all() diff --git a/tests/test_010_wells.py b/tests/test_010_wells.py index 70c382d2..bd07b143 100644 --- a/tests/test_010_wells.py +++ b/tests/test_010_wells.py @@ -3,7 +3,11 @@ import nlmod -def get_model_ds(): +def get_model_ds(time=None, start=None): + if time is None: + time = [1] + if start is None: + start = pd.Timestamp.today() kh = [10, 0.1, 20] kv = [0.5 * k for k in kh] @@ -18,7 +22,7 @@ def get_model_ds(): model_name="from_scratch", ) - ds = nlmod.time.set_ds_time(ds, time=pd.Timestamp.today()) + ds = nlmod.time.set_ds_time(ds, time=time, start=start) return ds @@ -46,6 +50,15 @@ def test_wel_from_df(): nlmod.gwf.wells.wel_from_df(wells, gwf) +def test_wel_from_df_no_multiplier(): + wells = pd.DataFrame(columns=["x", "y", "top", "botm", "Q"], index=range(2)) + wells.loc[0] = 100, -50, -5, -10, -100.0 + wells.loc[1] = 200, 150, -20, -30, -300.0 + + sim, gwf = get_sim_and_gwf() + nlmod.gwf.wells.wel_from_df(wells, gwf, auxmultname=None) + + def test_maw_from_df(): wells = pd.DataFrame(columns=["x", "y", "top", "botm", "rw", "Q"], index=range(2)) wells.loc[0] = 100, -50, -5, -10, 0.1, -100.0 @@ -53,3 +66,33 @@ def test_maw_from_df(): sim, gwf = get_sim_and_gwf() nlmod.gwf.wells.maw_from_df(wells, gwf) + + +def test_wel_from_df_transient(): + time = pd.date_range("2000", "2001", freq="MS") + ds = get_model_ds(start="1990", time=time) + + Q = pd.DataFrame(-100.0, index=time, columns=["Q1", "Q2"]) + wells = pd.DataFrame(columns=["x", "y", "top", "botm", "Q"], index=range(2)) + wells.loc[0] = 100, -50, -5, -10, "Q1" + wells.loc[1] = 200, 150, -20, -30, "Q2" + + sim, gwf = get_sim_and_gwf() + wel = nlmod.gwf.wells.wel_from_df(wells, gwf) + + nlmod.time.dataframe_to_flopy_timeseries(Q, ds, package=wel) + + +def test_maw_from_df_transient(): + time = pd.date_range("2000", "2001", freq="MS") + ds = get_model_ds(start="1990", time=time) + + Q = pd.DataFrame(-100.0, index=time, columns=["Q1", "Q2"]) + wells = pd.DataFrame(columns=["x", "y", "top", "botm", "rw", "Q"], index=range(2)) + wells.loc[0] = 100, -50, -5, -10, 0.1, "Q1" + wells.loc[1] = 200, 150, -20, -30, 0.1, "Q2" + + sim, gwf = get_sim_and_gwf() + maw = nlmod.gwf.wells.maw_from_df(wells, gwf) + + nlmod.time.dataframe_to_flopy_timeseries(Q, ds, package=maw) diff --git a/tests/test_011_dcs.py b/tests/test_011_dcs.py index affc3ce2..b7d0793f 100644 --- a/tests/test_011_dcs.py +++ b/tests/test_011_dcs.py @@ -8,6 +8,7 @@ def test_dcs_structured(): line = [(0, 0), (1000, 1000)] dcs = nlmod.plot.DatasetCrossSection(ds, line) dcs.plot_layers() + dcs.label_layers() dcs.plot_array(ds["kh"], alpha=0.5) dcs.plot_grid() @@ -17,5 +18,6 @@ def test_dcs_vertex(): line = [(0, 0), (1000, 1000)] dcs = nlmod.plot.DatasetCrossSection(ds, line) dcs.plot_layers() + dcs.label_layers() dcs.plot_array(ds["kh"], alpha=0.5) - dcs.plot_grid() + dcs.plot_grid(vertical=False) diff --git a/tests/test_013_surface_water.py b/tests/test_013_surface_water.py index 045a4b29..d05ed8d0 100644 --- a/tests/test_013_surface_water.py +++ b/tests/test_013_surface_water.py @@ -1,7 +1,7 @@ import os -import pandas as pd import geopandas as gpd +import pandas as pd import nlmod @@ -12,7 +12,7 @@ def test_gdf_to_seasonal_pkg(): ds = nlmod.get_ds( [170000, 171000, 550000, 551000], model_ws=model_ws, model_name=model_name ) - ds = nlmod.time.set_ds_time(ds, time=pd.Timestamp.today()) + ds = nlmod.time.set_ds_time(ds, time=[365.0], start=pd.Timestamp.today()) gdf = nlmod.gwf.surface_water.get_gdf(ds) sim = nlmod.sim.sim(ds) @@ -33,7 +33,7 @@ def test_gdf_lake(): ds = nlmod.get_ds( [170000, 171000, 550000, 551000], model_ws=model_ws, model_name=model_name ) - ds = nlmod.time.set_ds_time(ds, time=pd.Timestamp.today()) + ds = nlmod.time.set_ds_time(ds, time=[1], start=pd.Timestamp.today()) ds = nlmod.dims.refine(ds) sim = nlmod.sim.sim(ds) @@ -42,45 +42,44 @@ def test_gdf_lake(): gwf = nlmod.gwf.gwf(ds, sim) nlmod.gwf.dis(ds, gwf) - ds['evap'] = (('time',), [0.0004]) + ds["evap"] = (("time",), [0.0004]) # add lake with outlet and evaporation gdf_lake = gpd.GeoDataFrame( { "name": ["0", "0", "1"], - "lakeno": [0, 0, 1], "strt": [1.0, 1.0, 2.0], "clake": [10.0, 10.0, 10.0], - 'EVAPORATION': ['evap', 'evap', 'evap'], + "EVAPORATION": ["evap", "evap", "evap"], "lakeout": [1, 1, None], "outlet_invert": ["use_elevation", "use_elevation", None], }, index=[14, 15, 16], ) - nlmod.gwf.lake_from_gdf( - gwf, gdf_lake, ds, boundname_column="name", recharge=False) - - # remove lake package - gwf.remove_package('LAK_0') + nlmod.gwf.lake_from_gdf(gwf, gdf_lake, ds, boundname_column="name", recharge=False) + # remove lake package + gwf.remove_package("LAK_0") # add lake with outlet and inflow - ds['inflow'] = (('time',), [100.]) + ds["inflow"] = (("time",), [100.0]) gdf_lake = gpd.GeoDataFrame( { "name": ["0", "0", "1"], "lakeno": [0, 0, 1], "strt": [1.0, 1.0, 2.0], "clake": [10.0, 10.0, 10.0], - 'INFLOW': ['inflow', 'inflow', None], - "lakeout": [1, 1, -1], # lake 0 overflows in lake 1, the outlet from lake 1 is removed from the model + "INFLOW": ["inflow", "inflow", None], + "lakeout": [ + 1, + 1, + -1, + ], # lake 0 overflows in lake 1, the outlet from lake 1 is removed from the model "outlet_invert": [0, 0, None], }, index=[14, 15, 16], ) - nlmod.gwf.lake_from_gdf( - gwf, gdf_lake, ds, boundname_column="name", recharge=False) - + nlmod.gwf.lake_from_gdf(gwf, gdf_lake, ds, boundname_column="name", recharge=False) diff --git a/tests/test_015_gwf_output.py b/tests/test_015_gwf_output.py index d7efa9ba..8e31eba3 100644 --- a/tests/test_015_gwf_output.py +++ b/tests/test_015_gwf_output.py @@ -2,15 +2,21 @@ import tempfile import numpy as np +import pytest import test_001_model import nlmod from nlmod.dims.grid import refine -from nlmod.gwf import get_heads_da +from nlmod.gwf import get_budget_da, get_heads_da tmpdir = tempfile.gettempdir() tst_model_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") +grberror = ( + "Please provide grid information by passing path to the " + "binary grid file with `grbfile=`." +) + def test_create_small_model_grid_only(tmpdir, model_name="test"): model_name = "test" @@ -25,13 +31,7 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): ) assert ds.dims["layer"] == 5 - ds = nlmod.time.set_ds_time( - ds, - start_time="2015-1-1", - steady_state=False, - steady_start=True, - transient_timesteps=2, - ) + ds = nlmod.time.set_ds_time(ds, time=[1, 2, 3], start="2015-1-1", steady=[1, 0, 0]) # create simulation sim = nlmod.sim.sim(ds) @@ -49,13 +49,13 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): nlmod.gwf.dis(ds, gwf) # create node property flow - nlmod.gwf.npf(ds, gwf) + nlmod.gwf.npf(ds, gwf, save_flows=True) # Create the initial conditions package nlmod.gwf.ic(ds, gwf, starting_head=1.0) nlmod.gwf.oc(ds, gwf) - ds.update(nlmod.grid.mask_model_edge(ds, ds["idomain"])) + ds.update(nlmod.grid.mask_model_edge(ds)) nlmod.gwf.chd(ds, gwf, mask="edge_mask", head="starting_head") nlmod.sim.write_and_run(sim, ds) @@ -63,19 +63,22 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): heads_correct = np.ones((3, 5, 2, 3)) heads_correct[:, 3, :, 1:] = np.nan - da = get_heads_da(ds=ds, gwf=None, fname_hds=None) + da = get_heads_da(ds=ds, gwf=None, fname=None) # ds assert np.array_equal(da.values, heads_correct, equal_nan=True) - da = get_heads_da(ds=None, gwf=gwf, fname_hds=None) + da = get_heads_da(ds=None, gwf=gwf, fname=None) # gwf assert np.array_equal(da.values, heads_correct, equal_nan=True) fname_hds = os.path.join(ds.model_ws, ds.model_name + ".hds") - da = get_heads_da(ds=ds, gwf=None, fname_hds=fname_hds) + grbfile = os.path.join(ds.model_ws, ds.model_name + ".dis.grb") + da = get_heads_da(ds=None, gwf=None, fname=fname_hds, grbfile=grbfile) # fname assert np.array_equal(da.values, heads_correct, equal_nan=True) - fname_hds = os.path.join(ds.model_ws, ds.model_name + ".hds") - da = get_heads_da(ds=None, gwf=gwf, fname_hds=fname_hds) - assert np.array_equal(da.values, heads_correct, equal_nan=True) + # budget + da = get_budget_da("CHD", ds=ds, gwf=None, fname=None) # ds + da = get_budget_da("CHD", ds=None, gwf=gwf, fname=None) # gwf + fname_cbc = os.path.join(ds.model_ws, ds.model_name + ".cbc") + get_budget_da("CHD", ds=None, gwf=None, fname=fname_cbc, grbfile=grbfile) # fname # unstructured ds_unstr = refine( @@ -109,7 +112,7 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): nlmod.gwf.ic(ds_unstr, gwf_unstr, starting_head=1.0) nlmod.gwf.oc(ds_unstr, gwf_unstr) - ds_unstr.update(nlmod.grid.mask_model_edge(ds_unstr, ds_unstr["idomain"])) + ds_unstr.update(nlmod.grid.mask_model_edge(ds_unstr)) nlmod.gwf.chd(ds_unstr, gwf_unstr, mask="edge_mask", head="starting_head") nlmod.sim.write_and_run(sim, ds_unstr) @@ -117,19 +120,71 @@ def test_create_small_model_grid_only(tmpdir, model_name="test"): heads_correct = np.ones((3, 5, 6)) heads_correct[:, 3, [1, 2, 4, 5]] = np.nan - da = get_heads_da(ds=ds_unstr, gwf=None, fname_hds=None) + da = get_heads_da(ds=ds_unstr, gwf=None, fname=None) # ds assert np.array_equal(da.values, heads_correct, equal_nan=True) - da = get_heads_da(ds=None, gwf=gwf_unstr, fname_hds=None) + da = get_heads_da(ds=None, gwf=gwf_unstr, fname=None) # gwf assert np.array_equal(da.values, heads_correct, equal_nan=True) fname_hds = os.path.join(ds.model_ws, ds.model_name + ".hds") - da = get_heads_da(ds=ds_unstr, gwf=None, fname_hds=fname_hds) + grbfile = os.path.join(ds.model_ws, ds.model_name + ".disv.grb") + da = get_heads_da(ds=None, gwf=None, fname=fname_hds, grbfile=grbfile) # fname assert np.array_equal(da.values, heads_correct, equal_nan=True) - fname_hds = os.path.join(ds.model_ws, ds.model_name + ".hds") - da = get_heads_da(ds=None, gwf=gwf_unstr, fname_hds=fname_hds) - assert np.array_equal(da.values, heads_correct, equal_nan=True) + # budget + da = get_budget_da("CHD", ds=ds_unstr, gwf=None, fname=None) # ds + da = get_budget_da("CHD", ds=None, gwf=gwf_unstr, fname=None) # gwf + da = get_budget_da( + "CHD", ds=None, gwf=None, fname=fname_cbc, grbfile=grbfile + ) # fname + + +def test_get_heads_da_from_file_structured_no_grb(): + fname_hds = "./tests/data/mf6output/structured/test.hds" + with pytest.warns(UserWarning): + nlmod.gwf.output.get_heads_da(fname=fname_hds) + + +def test_get_heads_da_from_file_structured_with_grb(): + fname_hds = "./tests/data/mf6output/structured/test.hds" + grbfile = "./tests/data/mf6output/structured/test.dis.grb" + nlmod.gwf.output.get_heads_da(fname=fname_hds, grbfile=grbfile) + + +def test_get_budget_da_from_file_structured_no_grb(): + fname_cbc = "./tests/data/mf6output/structured/test.cbc" + with pytest.raises(ValueError, match=grberror): + nlmod.gwf.output.get_budget_da("CHD", fname=fname_cbc) + + +def test_get_budget_da_from_file_structured_with_grb(): + fname_cbc = "./tests/data/mf6output/structured/test.cbc" + grbfile = "./tests/data/mf6output/structured/test.dis.grb" + nlmod.gwf.output.get_budget_da("CHD", fname=fname_cbc, grbfile=grbfile) + + +def test_get_heads_da_from_file_vertex_no_grb(): + fname_hds = "./tests/data/mf6output/vertex/test.hds" + with pytest.warns(UserWarning): + nlmod.gwf.output.get_heads_da(fname=fname_hds) + + +def test_get_heads_da_from_file_vertex_with_grb(): + fname_hds = "./tests/data/mf6output/vertex/test.hds" + grbfile = "./tests/data/mf6output/vertex/test.disv.grb" + nlmod.gwf.output.get_heads_da(fname=fname_hds, grbfile=grbfile) + + +def test_get_budget_da_from_file_vertex_no_grb(): + fname_cbc = "./tests/data/mf6output/vertex/test.cbc" + with pytest.raises(ValueError, match=grberror): + nlmod.gwf.output.get_budget_da("CHD", fname=fname_cbc) + + +def test_get_budget_da_from_file_vertex_with_grb(): + fname_cbc = "./tests/data/mf6output/vertex/test.cbc" + grbfile = "./tests/data/mf6output/vertex/test.disv.grb" + nlmod.gwf.output.get_budget_da("CHD", fname=fname_cbc, grbfile=grbfile) def test_gxg(): diff --git a/tests/test_016_time.py b/tests/test_016_time.py index a606e727..1e3339c5 100644 --- a/tests/test_016_time.py +++ b/tests/test_016_time.py @@ -1,3 +1,5 @@ +import numpy as np + import nlmod @@ -17,3 +19,12 @@ def test_estimate_nstp(): assert nstp[-1] == nstp_min assert max(nstp) == nstp_max assert min(nstp) == nstp_min + + +def test_ds_time_from_tdis_settings(): + tidx = nlmod.time.ds_time_idx_from_tdis_settings( + "2000", [100, 100, 100], nstp=[1, 2, 2], tsmult=[1.0, 1.0, 2.0] + ) + + elapsed = (tidx.to_numpy() - np.datetime64("2000")) / np.timedelta64(1, "D") + assert np.allclose(elapsed, [100, 150, 200, 233.33333333, 300.0]) diff --git a/tests/test_019_attributes_encodings.py b/tests/test_019_attributes_encodings.py new file mode 100644 index 00000000..8e18ce38 --- /dev/null +++ b/tests/test_019_attributes_encodings.py @@ -0,0 +1,96 @@ +import os +import time +from tempfile import TemporaryDirectory + +import numpy as np +import xarray as xr + +from nlmod.dims.attributes_encodings import get_encodings + + +def test_encodings_float_as_int16(): + """Test if the encodings are correct.""" + # Test is the encodings work for floats where degradation to int16 is allowed + heads_data = np.arange(1.0, 6.0) + heads_data[1] = np.nan + + por_data = np.linspace(0.0, 1.0, 5) + por_data[1] = np.nan + + ds = xr.Dataset( + data_vars=dict( + heads=xr.DataArray(data=heads_data), + porosity=xr.DataArray(data=por_data), + ) + ) + encodings = get_encodings( + ds, set_encoding_inplace=False, allowed_to_read_data_vars_for_minmax=True + ) + + assert encodings["heads"]["dtype"] == "int16", "dtype should be int16" + assert encodings["porosity"]["dtype"] == "int16", "dtype should be int16" + + # test writing to temporary netcdf file + with TemporaryDirectory() as tmpdir: + fp_test = os.path.join(tmpdir, "test2.nc") + ds.to_netcdf(fp_test, encoding=encodings) + + with xr.open_dataset(fp_test, mask_and_scale=True) as ds2: + ds2.load() + + dval_max = float(ds["heads"].max() - ds["heads"].min()) / (32766 + 32767) + + assert np.allclose( + ds["heads"].values, ds2["heads"].values, atol=dval_max, rtol=0.0, equal_nan=True + ) + assert np.all(np.isnan(ds["heads"]) == np.isnan(ds2["heads"])) + + # Test is the encodings work for floats where degradation to int16 is not allowed + data = np.arange(1.0, 1e6) + data[1] = np.nan + + ds = xr.Dataset(data_vars=dict(heads=xr.DataArray(data=data))) + encodings = get_encodings( + ds, set_encoding_inplace=False, allowed_to_read_data_vars_for_minmax=True + ) + assert encodings["heads"]["dtype"] == "float32", "dtype should be float32" + pass + + +def test_encondings_inplace(): + """Test if the encodings inplace are correct.""" + # Test is the encodings work for floats where degradation to int16 is allowed + data = np.arange(1.0, 5.0) + data[1] = np.nan + + ds = xr.Dataset(data_vars=dict(heads=xr.DataArray(data=data))) + ds_inplace = ds.copy(deep=True) + + encodings = get_encodings( + ds, set_encoding_inplace=False, allowed_to_read_data_vars_for_minmax=True + ) + get_encodings( + ds_inplace, + set_encoding_inplace=False, + allowed_to_read_data_vars_for_minmax=True, + ) + + # test writing to temporary netcdf file + with TemporaryDirectory() as tmpdir: + fp_test = os.path.join(tmpdir, "test2.nc") + ds.to_netcdf(fp_test, encoding=encodings) + + with xr.open_dataset(fp_test, mask_and_scale=True) as ds2: + ds2.load() + + fp_test_inplace = os.path.join(tmpdir, "test_inplace.nc") + ds_inplace.to_netcdf(fp_test_inplace) + + with xr.open_dataset(fp_test_inplace, mask_and_scale=True) as ds_inplace2: + ds_inplace2.load() + + assert np.allclose(ds2["heads"].values, ds_inplace2["heads"].values, equal_nan=True) + pass + + +test_encodings_float_as_int16() diff --git a/tests/test_020_uzf.py b/tests/test_020_uzf.py new file mode 100644 index 00000000..7ecd1c00 --- /dev/null +++ b/tests/test_020_uzf.py @@ -0,0 +1,51 @@ +import numpy as np +import pandas as pd +import util + +import nlmod + + +def test_uzf_structured(): + # %% create model Dataset + extent = [200_000, 202_000, 400_000, 403_000] + ds = util.get_ds_structured(extent, top=0, botm=np.linspace(-1, -10, 10)) + + time = pd.date_range("2022", "2023", freq="D") + ds = nlmod.time.set_ds_time(ds, start="2021", time=time) + + ds.update(nlmod.read.knmi.get_recharge(ds, method="separate")) + + # %% generate sim and gwf + # create simulation + sim = nlmod.sim.sim(ds) + + # create time discretisation + _ = nlmod.sim.tdis(ds, sim) + + # create groundwater flow model + gwf = nlmod.gwf.gwf(ds, sim) + + # create ims + _ = nlmod.sim.ims(sim) + + # Create discretization + _ = nlmod.gwf.dis(ds, gwf) + + # create node property flow + _ = nlmod.gwf.npf(ds, gwf) + + # Create the initial conditions package + _ = nlmod.gwf.ic(ds, gwf, starting_head=1.0) + + # Create the output control package + _ = nlmod.gwf.oc(ds, gwf) + + bhead = ds["botm"][1] + cond = ds["area"] * 1 + _ = nlmod.gwf.ghb(ds, gwf, bhead=bhead, cond=cond, layer=len(ds.layer) - 1) + + # create recharge package + _ = nlmod.gwf.uzf(ds, gwf) + + # %% run + # _ = nlmod.sim.write_and_run(sim, ds) diff --git a/tests/test_021_nhi.py b/tests/test_021_nhi.py new file mode 100644 index 00000000..af768339 --- /dev/null +++ b/tests/test_021_nhi.py @@ -0,0 +1,22 @@ +import os +import numpy as np +import tempfile +import nlmod +import pytest + +tmpdir = tempfile.gettempdir() + + +@pytest.mark.slow +def test_buidrainage(): + model_ws = os.path.join(tmpdir, "buidrain") + ds = nlmod.get_ds([110_000, 130_000, 435_000, 445_000], model_ws=model_ws) + ds = nlmod.read.nhi.add_buisdrainage(ds) + + # assert that all locations with a specified depth also have a positive conductance + mask = ~ds["buisdrain_depth"].isnull() + assert np.all(ds["buisdrain_cond"].data[mask] > 0) + + # assert that all locations with a positive conductance also have a specified depth + mask = ds["buisdrain_cond"] > 0 + assert np.all(~np.isnan(ds["buisdrain_depth"].data[mask]))