WIP: Zarr backend

.travis.yml

@@ -43,6 +43,8 @@ matrix:
     env: CONDA_ENV=py36-pynio-dev
   - python: 3.6
     env: CONDA_ENV=py36-rasterio1.0alpha
+  - python: 3.6
+    env: CONDA_ENV=py36-zarr-dev
   allow_failures:
   - python: 3.6
     env:
@@ -67,6 +69,8 @@ matrix:
     env: CONDA_ENV=py36-pynio-dev
   - python: 3.6
     env: CONDA_ENV=py36-rasterio1.0alpha
+  - python: 3.6
+    env: CONDA_ENV=py36-zarr-dev
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then

ci/requirements-py27-cdat+pynio.yml

@@ -22,6 +22,7 @@ dependencies:
   - seaborn
   - toolz
   - rasterio
+  - zarr
   - pip:
     - coveralls
     - pytest-cov

ci/requirements-py27-windows.yml

@@ -19,3 +19,4 @@ dependencies:
   - seaborn
   - toolz
   - rasterio
+  - zarr

ci/requirements-py35.yml

@@ -17,6 +17,7 @@ dependencies:
   - seaborn
   - toolz
   - rasterio
+  - zarr
   - pip:
     - coveralls
     - pytest-cov

ci/requirements-py36-windows.yml

@@ -16,3 +16,4 @@ dependencies:
   - seaborn
   - toolz
   - rasterio
+  - zarr

ci/requirements-py36-zarr-dev.yml

@@ -0,0 +1,20 @@
+name: test_env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.6
+  - dask
+  - distributed
+  - matplotlib
+  - pytest
+  - flake8
+  - numpy
+  - pandas
+  - scipy
+  - seaborn
+  - toolz
+  - bottleneck
+  - pip:
+    - coveralls
+    - pytest-cov
+    - git+https://github.com/alimanfoo/zarr.git

ci/requirements-py36.yml

@@ -18,6 +18,7 @@ dependencies:
   - toolz
   - rasterio
   - bottleneck
+  - zarr
   - pip:
     - coveralls
     - pytest-cov

doc/api.rst

@@ -415,7 +415,9 @@ Dataset methods
    open_dataset
    open_mfdataset
    open_rasterio
+   open_zarr
    Dataset.to_netcdf
+   Dataset.to_zarr
    save_mfdataset
    Dataset.to_array
    Dataset.to_dataframe

doc/installing.rst

@@ -24,6 +24,7 @@ For netCDF and IO
   reading and writing netCDF4 files that does not use the netCDF-C libraries
 - `pynio <https://www.pyngl.ucar.edu/Nio.shtml>`__: for reading GRIB and other
   geoscience specific file formats
+- `zarr <http://zarr.readthedocs.io/`__: for chunked, compressed, N-dimensional arrays.
 
 For accelerating xarray
 ~~~~~~~~~~~~~~~~~~~~~~~

doc/io.rst

@@ -502,6 +502,17 @@ longitudes and latitudes.
 .. _test files: https://github.com/mapbox/rasterio/blob/master/tests/data/RGB.byte.tif
 .. _pyproj: https://github.com/jswhit/pyproj
 
+Zarr
+----
+
+`Zarr`_ is a Python package providing an implementation of chunked, compressed,
+N-dimensional arrays.
+
+TODO: fill in these docs
+
+.. _Zarr: http://zarr.readthedocs.io/
+
+
 .. _io.pynio:
 
 Formats supported by PyNIO
@@ -529,6 +540,7 @@ exporting your objects to pandas and using its broad range of `IO tools`_.
 .. _IO tools: http://pandas.pydata.org/pandas-docs/stable/io.html
 
 
+
 Combining multiple files
 ------------------------
 

xarray/__init__.py

@@ -18,6 +18,7 @@
 from .backends.api import (open_dataset, open_dataarray, open_mfdataset,
                            save_mfdataset)
 from .backends.rasterio_ import open_rasterio
+from .backends.zarr import open_zarr
 
 from .conventions import decode_cf, SerializationWarning
 

xarray/backends/__init__.py

@@ -10,3 +10,4 @@
 from .pynio_ import NioDataStore
 from .scipy_ import ScipyDataStore
 from .h5netcdf_ import H5NetCDFStore
+from .zarr import ZarrStore

xarray/backends/api.py

@@ -713,3 +713,29 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None,
     finally:
         for store in stores:
             store.close()
+
+
+def to_zarr(dataset, store=None, mode='a', synchronizer=None, group=None,
+            encoding=None):
+    """This function creates an appropriate datastore for writing a dataset to
+    disk a zarr ztore
+
+    See `Dataset.to_zarr` for full API docs.
+    """
+    if isinstance(store, path_type):
+        store = str(store)
+    if encoding is None:
+        encoding = {}
+
+    # validate Dataset keys, DataArray names, and attr keys/values
+    _validate_dataset_names(dataset)
+    _validate_attrs(dataset)
+
+    store = backends.ZarrStore(store=store, mode=mode,
+                               synchronizer=synchronizer, group=group,
+                               writer=None)
+
+    # I think zarr stores should always be sync'd immediately
+    # TODO: figure out how to properly handle unlimited_dims
+    dataset.dump_to_store(store, sync=True, encoding=encoding)
+    return store