diff --git a/.gitignore b/.gitignore index c28eebef..ef65e195 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ site example_eo example_mri .mypy_cache -*.req \ No newline at end of file +*.req +polytope_python.egg-info \ No newline at end of file diff --git a/performance/performance_many_num_steps.py b/performance/performance_many_num_steps.py index b22f276d..eb47b0a0 100644 --- a/performance/performance_many_num_steps.py +++ b/performance/performance_many_num_steps.py @@ -12,9 +12,6 @@ # Create a dataarray with 3 labelled axes using different index types options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - # "step": {"type_change": "int"}, - # "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, "latitude": {"reverse": {True}}, } diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index d8038e29..95d1d651 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,8 +1,11 @@ +import argparse import logging from abc import ABC, abstractmethod -from typing import Any +from typing import Any, List, Literal, Optional, Union import xarray as xr +from conflator import ConfigModel, Conflator +from pydantic import ConfigDict from ...utility.combinatorics import validate_axes from ..datacube_axis import DatacubeAxis @@ -35,6 +38,7 @@ def __init__(self, axis_options=None, datacube_options=None): self.compressed_grid_axes = [] self.compressed_axes = [] self.merged_axes = [] + self.unwanted_path = {} @abstractmethod def get(self, requests: IndexTree) -> Any: @@ -50,15 +54,16 @@ def validate(self, axes): def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations + transformation_options = transformation_type_key final_axis_names = DatacubeAxisTransformation.get_final_axes( - name, transformation_type_key, transformation_options + name, transformation_type_key.name, transformation_options ) transformation = DatacubeAxisTransformation.create_transform( - name, transformation_type_key, transformation_options + name, transformation_type_key.name, transformation_options ) # do not compress merged axes - if transformation_type_key == "merge": + if transformation_type_key.name == "merge": self.merged_axes.append(name) self.merged_axes.append(final_axis_names) @@ -78,8 +83,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt if self._axes is None or axis_name not in self._axes.keys(): DatacubeAxis.create_standard(axis_name, values, self) # add transformation tag to axis, as well as transformation options for later - setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a - # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties + setattr(self._axes[axis_name], has_transform[transformation_type_key.name], True) # where has_transform is + # a factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties # add the specific transformation handled here to the relevant axes # Modify the axis to update with the tag @@ -87,7 +92,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt self._axes[axis_name].transformations.append(transformation) def _add_all_transformation_axes(self, options, name, values): - for transformation_type_key in options.keys(): + for transformation_type_key in options.transformations: if transformation_type_key != "cyclic": self.transformed_axes.append(name) self._create_axes(name, values, transformation_type_key, options) @@ -143,6 +148,52 @@ def remap_path(self, path: DatacubePath): path[key] = self._axes[key].remap([value, value])[0][0] return path + @staticmethod + def create_axes_config(axis_options): + class TransformationConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + name: str = "" + + class CyclicConfig(TransformationConfig): + name: Literal["cyclic"] + range: List[float] = [0] + + class MapperConfig(TransformationConfig): + name: Literal["mapper"] + type: str = "" + resolution: Union[int, List[int]] = 0 + axes: List[str] = [""] + local: Optional[List[float]] = None + + class ReverseConfig(TransformationConfig): + name: Literal["reverse"] + is_reverse: bool = False + + class TypeChangeConfig(TransformationConfig): + name: Literal["type_change"] + type: str = "int" + + class MergeConfig(TransformationConfig): + name: Literal["merge"] + other_axis: str = "" + linkers: List[str] = [""] + + action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig] + + class AxisConfig(ConfigModel): + axis_name: str = "" + transformations: list[action_subclasses_union] + + class Config(ConfigModel): + config: list[AxisConfig] = [] + + parser = argparse.ArgumentParser(allow_abbrev=False) + axis_config = Conflator(app_name="polytope", model=Config, cli=False, argparser=parser).load() + if axis_options.get("config"): + axis_config = Config(config=axis_options.get("config")) + + return axis_config + @staticmethod def create(datacube, axis_options: dict, datacube_options={}): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index a5c7e3c5..cf75753b 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -2,7 +2,6 @@ from copy import deepcopy from itertools import product -import numpy as np import pygribjump as pygj from ...utility.geometry import nearest_pt @@ -19,6 +18,7 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): logging.info("Created an FDB datacube with options: " + str(axis_options)) self.unwanted_path = {} + self.axis_options = Datacube.create_axes_config(axis_options).config partial_request = config # Find values in the level 3 FDB datacube @@ -31,7 +31,11 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() - options = axis_options.get(name, None) + options = None + for opt in self.axis_options: + if opt.axis_name == name: + options = opt + # options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) self.treated_axes.append(name) self.complete_axes.append(name) @@ -39,7 +43,11 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in self.treated_axes: - options = axis_options.get(name, None) + options = None + for opt in self.axis_options: + if opt.axis_name == name: + options = opt + # options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) @@ -65,8 +73,6 @@ def get(self, requests: IndexTree): interm_branch_tuple_values = [] for key in compressed_request[0].keys(): # remove the tuple of the request when we ask the fdb - - # TODO: here, would need to take care of axes that are merged and unmerged, which need to be carefully decompressed interm_branch_tuple_values.append(compressed_request[0][key]) request_combis = product(*interm_branch_tuple_values) @@ -92,7 +98,8 @@ def get(self, requests: IndexTree): # request[0][key] = request[0][key][0] # branch_tuple_combi = product(*interm_branch_tuple_values) # # TODO: now build the relevant requests from this and ask gj for them - # # TODO: then group the output values together to fit back with the original compressed request and continue + # # TODO: then group the output values together to fit back with the original compressed request + # # and continue # new_requests = [] # for combi in branch_tuple_combi: # new_request = {} diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index b00c5351..0211738a 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -3,7 +3,7 @@ import numpy as np import xarray as xr -from .datacube import Datacube, IndexTree +from .datacube import Datacube class XArrayDatacube(Datacube): @@ -11,60 +11,83 @@ class XArrayDatacube(Datacube): def __init__(self, dataarray: xr.DataArray, axis_options=None, datacube_options=None): super().__init__(axis_options, datacube_options) + if axis_options is None: + axis_options = {} + if datacube_options is None: + datacube_options = {} + self.axis_options = Datacube.create_axes_config(axis_options).config + self.axis_counter = 0 + self._axes = None self.dataarray = dataarray for name, values in dataarray.coords.variables.items(): + options = None + for opt in self.axis_options: + if opt.axis_name == name: + options = opt if name in dataarray.dims: - options = self.axis_options.get(name, None) self._check_and_add_axes(options, name, values) self.treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): - options = self.axis_options.get(name, None) self._check_and_add_axes(options, name, values) self.treated_axes.append(name) for name in dataarray.dims: if name not in self.treated_axes: - options = self.axis_options.get(name, None) + options = None + for opt in self.axis_options: + if opt.axis_name == name: + options = opt val = dataarray[name].values[0] self._check_and_add_axes(options, name, val) self.treated_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in self.treated_axes: - options = self.axis_options.get(name, None) + options = None + for opt in self.axis_options: + if opt.axis_name == name: + options = opt val = self._axes[name].type self._check_and_add_axes(options, name, val) - def get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - if len(path.items()) == self.axis_counter: - # TODO: need to undo the tuples in the path into actual paths with a single value that xarray can read - unmapped_path = {} - path_copy = deepcopy(path) - for key in path_copy: - axis = self._axes[key] - key_value_path = {key: path_copy[key]} - (key_value_path, path, unmapped_path) = axis.unmap_path_key(key_value_path, path, unmapped_path) - path.update(key_value_path) - - unmapped_path = {} - self.refit_path(path, unmapped_path, path) - for key in path: - path[key] = list(path[key]) - for key in unmapped_path: - if isinstance(unmapped_path[key], tuple): - unmapped_path[key] = list(unmapped_path[key]) - - subxarray = self.dataarray.sel(path, method="nearest") - subxarray = subxarray.sel(unmapped_path) - value = subxarray.values - key = subxarray.name - r.result = (key, value) + def get(self, requests, leaf_path=None, axis_counter=0): + if leaf_path is None: + leaf_path = {} + if requests.axis.name == "root": + for c in requests.children: + self.get(c, leaf_path, axis_counter + 1) + else: + key_value_path = {requests.axis.name: requests.values} + ax = requests.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) + leaf_path.update(key_value_path) + if len(requests.children) != 0: + # We are not a leaf and we loop over + for c in requests.children: + self.get(c, leaf_path, axis_counter + 1) else: - r.remove_branch() + if self.axis_counter != axis_counter: + requests.remove_branch() + else: + # We are at a leaf and need to assign value to it + leaf_path_copy = deepcopy(leaf_path) + unmapped_path = {} + self.refit_path(leaf_path_copy, unmapped_path, leaf_path) + for key in leaf_path_copy: + leaf_path_copy[key] = list(leaf_path_copy[key]) + for key in unmapped_path: + if isinstance(unmapped_path[key], tuple): + unmapped_path[key] = list(unmapped_path[key]) + subxarray = self.dataarray.sel(leaf_path_copy, method="nearest") + subxarray = subxarray.sel(unmapped_path) + # value = subxarray.item() + value = subxarray.values + key = subxarray.name + requests.result = (key, value) def datacube_natural_indexes(self, axis, subarray): if axis.name in self.complete_axes: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 8b54645e..a798b105 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -301,6 +301,8 @@ def serialize(self, value): np.datetime64: PandasTimestampDatacubeAxis(), np.timedelta64: PandasTimedeltaDatacubeAxis(), np.float64: FloatDatacubeAxis(), + np.float32: FloatDatacubeAxis(), + np.int32: IntDatacubeAxis(), np.str_: UnsliceableDatacubeAxis(), str: UnsliceableDatacubeAxis(), np.object_: UnsliceableDatacubeAxis(), diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index b0dd6e1d..b7e402ea 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -11,7 +11,7 @@ class DatacubeAxisCyclic(DatacubeAxisTransformation): def __init__(self, name, cyclic_options): self.name = name self.transformation_options = cyclic_options - self.range = cyclic_options + self.range = cyclic_options.range def generate_final_transformation(self): return self diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index a443e7a7..272139a1 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -9,12 +9,12 @@ class DatacubeMapper(DatacubeAxisTransformation): def __init__(self, name, mapper_options): self.transformation_options = mapper_options - self.grid_type = mapper_options["type"] - self.grid_resolution = mapper_options["resolution"] - self.grid_axes = mapper_options["axes"] + self.grid_type = mapper_options.type + self.grid_resolution = mapper_options.resolution + self.grid_axes = mapper_options.axes self.local_area = [] - if "local" in mapper_options.keys(): - self.local_area = mapper_options["local"] + if mapper_options.local is not None: + self.local_area = mapper_options.local self.old_axis = name self._final_transformation = self.generate_final_transformation() self._final_mapped_axes = self._final_transformation._mapped_axes diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py index bd34973e..1cf19663 100644 --- a/polytope/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -11,8 +11,8 @@ def __init__(self, name, merge_options): self.transformation_options = merge_options self.name = name self._first_axis = name - self._second_axis = merge_options["with"] - self._linkers = merge_options["linkers"] + self._second_axis = merge_options.other_axis + self._linkers = merge_options.linkers def blocked_axes(self): return [self._second_axis] @@ -33,7 +33,6 @@ def merged_values(self, datacube): first_val = first_ax_vals[i] for j in range(len(second_ax_vals)): second_val = second_ax_vals[j] - # TODO: check that the first and second val are strings val_to_add = pd.to_datetime("".join([first_val, linkers[0], second_val, linkers[1]])) val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index dabf4ff9..eb032515 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -14,7 +14,8 @@ def create_transform(name, transformation_type_key, transformation_options): file_name = ".datacube_" + transformation_file_name module = import_module("polytope.datacube.transformations" + file_name + file_name) constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options[transformation_type_key] + # transformation_type_option = transformation_options[transformation_type_key] + transformation_type_option = transformation_options new_transformation = deepcopy(constructor(name, transformation_type_option)) new_transformation.name = name diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py index c0c25cdd..6daa752b 100644 --- a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -10,7 +10,7 @@ class DatacubeAxisTypeChange(DatacubeAxisTransformation): def __init__(self, name, type_options): self.name = name self.transformation_options = type_options - self.new_type = type_options + self.new_type = type_options.type self._final_transformation = self.generate_final_transformation() def generate_final_transformation(self): diff --git a/polytope/polytope.py b/polytope/polytope.py index 717279ba..c23b4e65 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -65,7 +65,8 @@ def retrieve(self, request: Request, method="standard"): # TODO: remove grid axes from the possible compressed_axes all_datacube_coupled_axes = [] for coupled_axes in self.datacube.coupled_axes: - # NOTE: the last axis from the coupled axes can always be compressed? Causes problems to fetch data using pygribjump + # NOTE: the last axis from the coupled axes can always be compressed? Causes problems to fetch data + # using pygribjump all_datacube_coupled_axes.extend(coupled_axes) self.datacube.compressed_axes = [ ax for ax in self.datacube.compressed_axes if ax not in all_datacube_coupled_axes diff --git a/requirements.txt b/requirements.txt index afbacd4c..8c7b04c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ scipy sortedcontainers tripy xarray +conflator @ git+https://github.com/ecmwf/conflator.git@main diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 6196251e..ea274fe4 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -1,5 +1,7 @@ import pandas as pd +import yaml +from polytope.datacube.backends.datacube import Datacube from polytope.datacube.datacube_axis import ( DatacubeAxisCyclic, FloatDatacubeAxis, @@ -31,13 +33,25 @@ def test_float_axis(self): def test_float_axis_cyclic(self): axis = FloatDatacubeAxis() + axis.is_cyclic = True assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 assert axis.serialize(2.0) == 2.0 - transformation = DatacubeAxisCyclic("", {}) + options = yaml.safe_load( + """ + config: + - axis_name: long + transformations: + - name: "cyclic" + range: [0, 1.0] + """ + ) + transformation = Datacube.create_axes_config(options) + transformation_option = transformation.config[0].transformations[0] + transformation = DatacubeAxisCyclic("", transformation_option) # Test the to_intervals function transformation.range = [1, 3] assert transformation.to_intervals([4, 7], [[]], axis) == [[4, 5], [5, 7], [7, 7]] diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index f94bb719..0c76819c 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -21,8 +21,10 @@ def setup_method(self, method): }, ) options = { - "long": {"cyclic": [-1.1, -0.1]}, - "level": {"cyclic": [1, 129]}, + "config": [ + {"axis_name": "long", "transformations": [{"name": "cyclic", "range": [-1.1, -0.1]}]}, + {"axis_name": "level", "transformations": [{"name": "cyclic", "range": [1, 129]}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 36615cfd..032db0bd 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -21,8 +21,10 @@ def setup_method(self, method): }, ) self.options = { - "long": {"cyclic": [-1.1, -0.1]}, - "level": {"cyclic": [1, 129]}, + "config": [ + {"axis_name": "long", "transformations": [{"name": "cyclic", "range": [-1.1, -0.1]}]}, + {"axis_name": "level", "transformations": [{"name": "cyclic", "range": [1, 129]}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index b3ff2cb3..77d370d4 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -21,8 +21,10 @@ def setup_method(self, method): }, ) self.options = { - "long": {"cyclic": [0, 1.0]}, - "level": {"cyclic": [1, 129]}, + "config": [ + {"axis_name": "long", "transformations": [{"name": "cyclic", "range": [0, 1.0]}]}, + {"axis_name": "level", "transformations": [{"name": "cyclic", "range": [1, 129]}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 079c92a2..8b796ff4 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -18,12 +18,21 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index 802df5c1..bf7759f5 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -20,7 +20,12 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} + options = { + "config": [ + {"axis_name": "long", "transformations": [{"name": "cyclic", "range": [0, 1.0]}]}, + {"axis_name": "level", "transformations": [{"name": "cyclic", "range": [0, 129]}]}, + ] + } self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index 3845d87a..6f76ed25 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -15,7 +15,8 @@ def setup_method(self, method): "long": [0, 0.5, 1.0], }, ) - options = {"long": {"cyclic": [0, 1.0]}} + + options = {"config": [{"axis_name": "long", "transformations": [{"name": "cyclic", "range": [0, 1.0]}]}]} self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index b05c35ab..46394552 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -17,8 +17,16 @@ def setup_method(self, method): latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "latitude": {"reverse": {True}}, + "config": [ + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 43e0ac77..56a7c8c9 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -12,10 +12,21 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index b91b3fc6..97fa6ecd 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -15,11 +15,22 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 60f4ff6c..c2764f9a 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -15,8 +15,16 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.options = { - "values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}, - "longitude": {"cyclic": [0, 360]}, + "config": [ + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) @@ -31,7 +39,7 @@ def test_healpix_grid(self): Select("valid_time", ["2022-12-14T13:00:00"]), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 40 lats = [] diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 65487c73..6301fb07 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -14,11 +14,22 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": [" ", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index f5108f86..84dbaa73 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -12,19 +12,28 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "mapper": { - "type": "local_regular", - "resolution": [80, 80], - "axes": ["latitude", "longitude"], - "local": [-40, 40, -20, 60], - } - }, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [-180, 180]}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "local_regular", + "resolution": [80, 80], + "axes": ["latitude", "longitude"], + "local": [-40, 40, -20, 60], + } + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [-180, 180]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index 04cd4cab..8136e653 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -12,18 +12,27 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "mapper": { - "type": "local_regular", - "resolution": [80, 80], - "axes": ["latitude", "longitude"], - "local": [-40, 40, -20, 60], - } - }, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "local_regular", + "resolution": [80, 80], + "axes": ["latitude", "longitude"], + "local": [-40, 40, -20, 60], + } + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_swiss_grid.py b/tests/test_local_swiss_grid.py index 69d1984d..1c9487f5 100644 --- a/tests/test_local_swiss_grid.py +++ b/tests/test_local_swiss_grid.py @@ -15,20 +15,28 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "mapper": { - "type": "local_regular", - "resolution": [193, 417], - "axes": ["latitude", "longitude"], - "local": [45.485, 48.1, 5.28985, 10.9087], - } - }, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "levelist": {"type_change": "int"}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "levelist", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "local_regular", + "resolution": [193, 417], + "axes": ["latitude", "longitude"], + "local": [45.485, 48.1, 5.28985, 10.9087], + } + ], + }, + ] } - self.config = {"param": "3008"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index e6cde268..a325143f 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -22,9 +22,19 @@ def setup_method(self, method): ) self.options = { - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "step": {"cyclic": [0, 2]}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "cyclic", "type": [0, 2]}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index d56378d3..1b05947a 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -16,9 +16,16 @@ def setup_method(self, method): self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "longitude": {"cyclic": [0, 360.0]}, - "latitude": {"reverse": {True}}, + "config": [ + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 12d1f76c..02066378 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -18,7 +18,14 @@ def setup_method(self, method): "time": ["0600"], }, ) - self.options = {"date": {"merge": {"with": "time", "linkers": ["T", "00"]}}} + self.options = { + "config": [ + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + } + ] + } self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py index dd1df749..a5f10126 100644 --- a/tests/test_multiple_param_fdb.py +++ b/tests/test_multiple_param_fdb.py @@ -12,10 +12,21 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 131b996f..6a57d3d9 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -16,8 +16,15 @@ def setup_method(self, method): self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "latitude": {"reverse": {True}}, + "config": [ + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + ] } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index ddafb68a..d5a2d7c0 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -12,12 +12,22 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index fdbf5f12..eba2a584 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -17,11 +17,21 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/wave.grib" download_test_data(nexus_url, "wave.grib") self.options = { - "values": {"mapper": {"type": "reduced_ll", "resolution": 1441, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "reduced_ll", "resolution": 1441, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "stream": "wave"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 4fa1f7c8..ada77b7b 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -17,12 +17,22 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index 7e297c8d..ef935186 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -16,7 +16,7 @@ def setup_method(self, method): "lat": [4, 3, 2, 1], }, ) - options = {"lat": {"reverse": {True}}} + options = {"config": [{"axis_name": "lat", "transformations": [{"name": "reverse", "is_reverse": True}]}]} self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index cc7d45a8..5fa0d075 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Select, Span @@ -22,8 +21,9 @@ def setup_method(self, method): "longitude": range(0, 360), }, ) - self.xarraydatacube = XArrayDatacube(array) - self.options = {"longitude": {"cyclic": [0, 360]}} + self.options = { + "config": [{"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}] + } self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) @@ -46,12 +46,22 @@ def test_all_mapper_cyclic(self): from polytope.datacube.backends.fdb import FDBDatacube self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "11"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index fae61798..af537ad1 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -12,11 +12,22 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "number": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 52ff69d7..b0888d6f 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -13,10 +13,22 @@ def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, - "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - "step": {"type_change": "int"}, - "latitude": {"reverse": {True}}, + "config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "levelist", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -43,19 +55,8 @@ def test_fdb_datacube(self): ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 6 + assert len(result.leaves) == 3 path1 = result.leaves[0].flatten() - assert path1["date"] == ( - np.datetime64("2017-01-01T12:00:00"), - # np.datetime64("2017-01-02T00:00:00"), - # np.datetime64("2017-01-02T12:00:00"), - ) - assert path1["levelist"] == ("500",) - path1 = result.leaves[1].flatten() - assert path1["date"] == ( - np.datetime64("2017-01-01T12:00:00"), - # np.datetime64("2017-01-02T00:00:00"), - # np.datetime64("2017-01-02T12:00:00"), - ) - assert path1["levelist"] == ("850",) - assert len(result.leaves[0].result) == 1 + assert path1["date"] == (np.datetime64("2017-01-01T12:00:00"),) + assert set(path1["levelist"]) == set((850, 500)) + assert len(result.leaves[0].result) == 2 diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index b26587ca..ffccee26 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -16,7 +16,7 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t self.slicer = HullSlicer() - options = {"latitude": {"reverse": {True}}} + options = {"config": [{"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}]} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) @pytest.mark.internet diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 4b7bf8d6..7c3b15af 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -20,8 +20,10 @@ def setup_method(self, method): array = ds.to_xarray().isel(step=0).t self.slicer = HullSlicer() options = { - "latitude": {"reverse": {True}}, - "longitude": {"cyclic": [0, 360.0]}, + "config": [ + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ] } self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index aa8da306..40c3b2c6 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -17,7 +17,7 @@ def setup_method(self, method): }, ) self.array = array - options = {"step": {"type_change": "int"}} + options = {"config": [{"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}]} self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options)