diff --git a/.gitignore b/.gitignore index 5dbb14f0..54ea62f1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ polytope_venv polytope_venv_latest new_updated_numpy_venv newest-polytope-venv +serializedTree diff --git a/codecov.yml b/codecov.yml index 5c74f4da..f911473e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,9 @@ +codecov: + branch: develop # set new Default branch + ignore: - "tests" # ignore tests folder + - "**/test*" + - "**/pb2*" + - "**/fdb*" # ignore fdb backend which we can't test in CI yet - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index f17eefa7..710d7792 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ jinja2>=3.1.3 -Markdown<3.2 -# mkdocs>=1.3.0 \ No newline at end of file +Markdown \ No newline at end of file diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 450ed2a3..fd10a130 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -78,6 +78,7 @@ def check_branching_axes(self, request): self._axes.pop(axis_name, None) def get(self, requests: TensorIndexTree): + requests.pprint() if len(requests.children) == 0: return requests fdb_requests = [] diff --git a/polytope/datacube/tensor_index_tree.py b/polytope/datacube/tensor_index_tree.py index 70749ccf..5cb813b9 100644 --- a/polytope/datacube/tensor_index_tree.py +++ b/polytope/datacube/tensor_index_tree.py @@ -81,8 +81,12 @@ def __eq__(self, other): for i in range(len(other.values)): other_val = other.values[i] self_val = self.values[i] - if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): - return False + if self.axis.can_round: + if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): + return False + else: + if other_val != self_val: + return False return True def __lt__(self, other): @@ -217,7 +221,7 @@ def flatten(self): def get_ancestors(self): ancestors = [] current_node = self - while current_node.axis != TensorIndexTree.root: + while current_node.axis.name != "root": ancestors.append(current_node) current_node = current_node.parent return ancestors[::-1] diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index df9b78fb..767f8916 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -109,9 +109,6 @@ def remap_values(self, ax, value): return remapped_val def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_nodes, slice_axis_idx): - if len(values) == 0: - node.remove_branch() - for i, value in enumerate(values): if i == 0: fvalue = ax.to_float(value) @@ -140,8 +137,16 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_node def _build_branch(self, ax, node, datacube, next_nodes): if ax.name not in self.compressed_axes: + parent_node = node.parent + right_unsliced_polytopes = [] for polytope in node["unsliced_polytopes"]: if ax.name in polytope._axes: + right_unsliced_polytopes.append(polytope) + # for polytope in node["unsliced_polytopes"]: + for i, polytope in enumerate(right_unsliced_polytopes): + node._parent = parent_node + # if ax.name in polytope._axes: + if True: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is # NOTE: we should have already created the ax_is_unsliceable cache before @@ -149,13 +154,22 @@ def _build_branch(self, ax, node, datacube, next_nodes): self._build_unsliceable_child(polytope, ax, node, datacube, [lower], next_nodes, slice_axis_idx) else: values = self.find_values_between(polytope, ax, node, datacube, lower, upper) + # NOTE: need to only remove the branches if the values are empty, + # but only if there are no other possible children left in the tree that + # we can append and if somehow this happens before and we need to remove, then what do we do?? + if i == len(right_unsliced_polytopes) - 1: + # we have iterated all polytopes and we can now remove the node if we need to + if len(values) == 0 and len(node.children) == 0: + node.remove_branch() self._build_sliceable_child(polytope, ax, node, datacube, values, next_nodes, slice_axis_idx) else: all_values = [] all_lowers = [] first_polytope = False first_slice_axis_idx = False + parent_node = node.parent for polytope in node["unsliced_polytopes"]: + node._parent = parent_node if ax.name in polytope._axes: # keep track of the first polytope defined on the given axis if not first_polytope: @@ -173,6 +187,8 @@ def _build_branch(self, ax, node, datacube, next_nodes): first_polytope, ax, node, datacube, all_lowers, next_nodes, first_slice_axis_idx ) else: + if len(all_values) == 0: + node.remove_branch() self._build_sliceable_child( first_polytope, ax, node, datacube, all_values, next_nodes, first_slice_axis_idx ) diff --git a/polytope/options.py b/polytope/options.py index 7cce36b9..78c7a20d 100644 --- a/polytope/options.py +++ b/polytope/options.py @@ -6,54 +6,66 @@ from pydantic import ConfigDict +class TransformationConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + name: str = "" + + +class CyclicConfig(TransformationConfig): + name: Literal["cyclic"] + range: List[float] = [0] + + +class MapperConfig(TransformationConfig): + name: Literal["mapper"] + type: str = "" + resolution: Union[int, List[int]] = 0 + axes: List[str] = [""] + local: Optional[List[float]] = None + + +class ReverseConfig(TransformationConfig): + name: Literal["reverse"] + is_reverse: bool = False + + +class TypeChangeConfig(TransformationConfig): + name: Literal["type_change"] + type: str = "int" + + +class MergeConfig(TransformationConfig): + name: Literal["merge"] + other_axis: str = "" + linkers: List[str] = [""] + + +action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig] + + +class AxisConfig(ConfigModel): + axis_name: str = "" + transformations: list[action_subclasses_union] + + +path_subclasses_union = Union[str, int, float] + + +class GribJumpAxesConfig(ConfigModel): + axis_name: str = "" + values: List[str] = [""] + + +class Config(ConfigModel): + axis_config: List[AxisConfig] = [] + compressed_axes_config: List[str] = [""] + pre_path: Optional[Dict[str, path_subclasses_union]] = {} + alternative_axes: List[GribJumpAxesConfig] = [] + + class PolytopeOptions(ABC): @staticmethod def get_polytope_options(options): - class TransformationConfig(ConfigModel): - model_config = ConfigDict(extra="forbid") - name: str = "" - - class CyclicConfig(TransformationConfig): - name: Literal["cyclic"] - range: List[float] = [0] - - class MapperConfig(TransformationConfig): - name: Literal["mapper"] - type: str = "" - resolution: Union[int, List[int]] = 0 - axes: List[str] = [""] - local: Optional[List[float]] = None - - class ReverseConfig(TransformationConfig): - name: Literal["reverse"] - is_reverse: bool = False - - class TypeChangeConfig(TransformationConfig): - name: Literal["type_change"] - type: str = "int" - - class MergeConfig(TransformationConfig): - name: Literal["merge"] - other_axis: str = "" - linkers: List[str] = [""] - - action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig] - - class AxisConfig(ConfigModel): - axis_name: str = "" - transformations: list[action_subclasses_union] - - path_subclasses_union = Union[str, int, float] - - class GribJumpAxesConfig(ConfigModel): - axis_name: str = "" - values: List[str] = [""] - - class Config(ConfigModel): - axis_config: List[AxisConfig] = [] - compressed_axes_config: List[str] = [""] - pre_path: Optional[Dict[str, path_subclasses_union]] = {} - alternative_axes: List[GribJumpAxesConfig] = [] parser = argparse.ArgumentParser(allow_abbrev=False) conflator = Conflator(app_name="polytope", model=Config, cli=False, argparser=parser, **options) diff --git a/polytope/version.py b/polytope/version.py index 68cdeee4..382021f3 100644 --- a/polytope/version.py +++ b/polytope/version.py @@ -1 +1 @@ -__version__ = "1.0.5" +__version__ = "1.0.6" diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index bcab5199..80d5c96e 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -73,6 +73,96 @@ def test_fdb_datacube(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 + @pytest.mark.fdb + def test_fdb_datacube_select_non_existing_last(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 3 + + @pytest.mark.fdb + def test_fdb_datacube_select_non_existing_first(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 3 + + @pytest.mark.fdb + def test_fdb_datacube_select_completely_non_existing(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230626T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 0 + @pytest.mark.fdb def test_fdb_datacube_disk(self): import pygribjump as gj