From 0452a122730a01f3138e14773c7a1422ecfa844b Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 5 Aug 2024 14:53:33 +0200 Subject: [PATCH 01/10] fix undeterministic ufunc error for timestamp axis --- polytope/datacube/tensor_index_tree.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/tensor_index_tree.py b/polytope/datacube/tensor_index_tree.py index 70749ccf..c3619b28 100644 --- a/polytope/datacube/tensor_index_tree.py +++ b/polytope/datacube/tensor_index_tree.py @@ -81,8 +81,9 @@ def __eq__(self, other): for i in range(len(other.values)): other_val = other.values[i] self_val = self.values[i] - if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): - return False + if self.axis.can_round: + if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): + return False return True def __lt__(self, other): From d349b98e7f805068602cedfab3068cb6a4ff4e19 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 08:45:18 +0200 Subject: [PATCH 02/10] fix bug where multiple select, some of which don't exist --- polytope/datacube/backends/fdb.py | 1 + polytope/datacube/tensor_index_tree.py | 16 +++++++++---- polytope/engine/hullslicer.py | 33 ++++++++++++++++++++++++-- tests/test_slice_date_range_fdb.py | 32 +++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 450ed2a3..fd10a130 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -78,6 +78,7 @@ def check_branching_axes(self, request): self._axes.pop(axis_name, None) def get(self, requests: TensorIndexTree): + requests.pprint() if len(requests.children) == 0: return requests fdb_requests = [] diff --git a/polytope/datacube/tensor_index_tree.py b/polytope/datacube/tensor_index_tree.py index c3619b28..a282e53d 100644 --- a/polytope/datacube/tensor_index_tree.py +++ b/polytope/datacube/tensor_index_tree.py @@ -81,9 +81,8 @@ def __eq__(self, other): for i in range(len(other.values)): other_val = other.values[i] self_val = self.values[i] - if self.axis.can_round: - if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): - return False + if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): + return False return True def __lt__(self, other): @@ -210,6 +209,9 @@ def remove_compressed_branch(self, value): def flatten(self): path = DatacubePath() + # print("SELF IN FLATTEN") + # print(self) + # print(self.parent) ancestors = self.get_ancestors() for ancestor in ancestors: path[ancestor.axis.name] = ancestor.values @@ -217,8 +219,14 @@ def flatten(self): def get_ancestors(self): ancestors = [] + # print("SELF IN GET ANCESTORS") + # print(self) + # print(self.axis) current_node = self - while current_node.axis != TensorIndexTree.root: + # while current_node.axis != TensorIndexTree.root: + while current_node.axis.name != "root": ancestors.append(current_node) current_node = current_node.parent + # print(current_node) + # print(current_node.axis.name == "root") return ancestors[::-1] diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index df9b78fb..ad09de93 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -73,6 +73,10 @@ def find_values_between(self, polytope, ax, node, datacube, lower, upper): tol = ax.tol lower = ax.from_float(lower - tol) upper = ax.from_float(upper + tol) + # print(node) + # print("WHAT S THE AXIS") + # print(node.axis) + # print(node.flatten()) flattened = node.flatten() method = polytope.method if method == "nearest": @@ -109,8 +113,12 @@ def remap_values(self, ax, value): return remapped_val def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_nodes, slice_axis_idx): - if len(values) == 0: - node.remove_branch() + # print(node.children) + # if len(values) == 0 and len(node.children) == 0: + # print(node) + # print(values) + # if len(values) == 0: + # node.remove_branch() for i, value in enumerate(values): if i == 0: @@ -140,22 +148,41 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_node def _build_branch(self, ax, node, datacube, next_nodes): if ax.name not in self.compressed_axes: + parent_node = node.parent + right_unsliced_polytopes = [] for polytope in node["unsliced_polytopes"]: if ax.name in polytope._axes: + right_unsliced_polytopes.append(polytope) + # for polytope in node["unsliced_polytopes"]: + for i, polytope in enumerate(right_unsliced_polytopes): + node._parent = parent_node + # if ax.name in polytope._axes: + if True: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is # NOTE: we should have already created the ax_is_unsliceable cache before if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, [lower], next_nodes, slice_axis_idx) else: + # print("IN BUILD BRANCH") + # print(node) + # print(node.axis) + # print(node.parent) values = self.find_values_between(polytope, ax, node, datacube, lower, upper) + # TODO: HERE, SOMEHOW NEED TO ONLY remove the branches if the values are empty, but only if there are no other possible children left in the tree that we can append and if somehow this happens before and we need to remove, then what do we do?? + if i == len(right_unsliced_polytopes) - 1: + # we have iterated all polytopes and we can now remove the node if we need to + if len(values) == 0 and len(node.children) == 0: + node.remove_branch() self._build_sliceable_child(polytope, ax, node, datacube, values, next_nodes, slice_axis_idx) else: all_values = [] all_lowers = [] first_polytope = False first_slice_axis_idx = False + parent_node = node.parent for polytope in node["unsliced_polytopes"]: + node._parent = parent_node if ax.name in polytope._axes: # keep track of the first polytope defined on the given axis if not first_polytope: @@ -173,6 +200,8 @@ def _build_branch(self, ax, node, datacube, next_nodes): first_polytope, ax, node, datacube, all_lowers, next_nodes, first_slice_axis_idx ) else: + if len(all_values) == 0: + node.remove_branch() self._build_sliceable_child( first_polytope, ax, node, datacube, all_values, next_nodes, first_slice_axis_idx ) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index bcab5199..0ebbbc26 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -72,6 +72,38 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 + + + @pytest.mark.fdb + def test_fdb_datacube_select(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 3 + @pytest.mark.fdb def test_fdb_datacube_disk(self): From 26bdb3d6ba3a1ad1255c2d09f30e1d7e5df2169c Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 09:00:35 +0200 Subject: [PATCH 03/10] test more --- tests/test_slice_date_range_fdb.py | 62 +++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 0ebbbc26..80d5c96e 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -72,10 +72,9 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - @pytest.mark.fdb - def test_fdb_datacube_select(self): + def test_fdb_datacube_select_non_existing_last(self): import pygribjump as gj request = Request( @@ -104,6 +103,65 @@ def test_fdb_datacube_select(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 + @pytest.mark.fdb + def test_fdb_datacube_select_non_existing_first(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 3 + + @pytest.mark.fdb + def test_fdb_datacube_select_completely_non_existing(self): + import pygribjump as gj + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230626T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + + self.fdbdatacube = gj.GribJump() + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + options=self.options, + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + for i in range(len(result.leaves)): + assert len(result.leaves[i].result) == 0 @pytest.mark.fdb def test_fdb_datacube_disk(self): From da3248d0627ee27afc6e7d55a3cd2593551ebd76 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 10:03:23 +0200 Subject: [PATCH 04/10] fix comparing time values with floats for comparison --- polytope/datacube/tensor_index_tree.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/tensor_index_tree.py b/polytope/datacube/tensor_index_tree.py index a282e53d..15d8c59f 100644 --- a/polytope/datacube/tensor_index_tree.py +++ b/polytope/datacube/tensor_index_tree.py @@ -81,8 +81,12 @@ def __eq__(self, other): for i in range(len(other.values)): other_val = other.values[i] self_val = self.values[i] - if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): - return False + if self.axis.can_round: + if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol): + return False + else: + if other_val != self_val: + return False return True def __lt__(self, other): From de849f7e3bcd8810e56d7fe51318d46cade7dded Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 10:16:47 +0200 Subject: [PATCH 05/10] clean up --- polytope/datacube/tensor_index_tree.py | 9 --------- polytope/engine/hullslicer.py | 19 +++---------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/polytope/datacube/tensor_index_tree.py b/polytope/datacube/tensor_index_tree.py index 15d8c59f..5cb813b9 100644 --- a/polytope/datacube/tensor_index_tree.py +++ b/polytope/datacube/tensor_index_tree.py @@ -213,9 +213,6 @@ def remove_compressed_branch(self, value): def flatten(self): path = DatacubePath() - # print("SELF IN FLATTEN") - # print(self) - # print(self.parent) ancestors = self.get_ancestors() for ancestor in ancestors: path[ancestor.axis.name] = ancestor.values @@ -223,14 +220,8 @@ def flatten(self): def get_ancestors(self): ancestors = [] - # print("SELF IN GET ANCESTORS") - # print(self) - # print(self.axis) current_node = self - # while current_node.axis != TensorIndexTree.root: while current_node.axis.name != "root": ancestors.append(current_node) current_node = current_node.parent - # print(current_node) - # print(current_node.axis.name == "root") return ancestors[::-1] diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index ad09de93..767f8916 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -73,10 +73,6 @@ def find_values_between(self, polytope, ax, node, datacube, lower, upper): tol = ax.tol lower = ax.from_float(lower - tol) upper = ax.from_float(upper + tol) - # print(node) - # print("WHAT S THE AXIS") - # print(node.axis) - # print(node.flatten()) flattened = node.flatten() method = polytope.method if method == "nearest": @@ -113,13 +109,6 @@ def remap_values(self, ax, value): return remapped_val def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_nodes, slice_axis_idx): - # print(node.children) - # if len(values) == 0 and len(node.children) == 0: - # print(node) - # print(values) - # if len(values) == 0: - # node.remove_branch() - for i, value in enumerate(values): if i == 0: fvalue = ax.to_float(value) @@ -164,12 +153,10 @@ def _build_branch(self, ax, node, datacube, next_nodes): if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, [lower], next_nodes, slice_axis_idx) else: - # print("IN BUILD BRANCH") - # print(node) - # print(node.axis) - # print(node.parent) values = self.find_values_between(polytope, ax, node, datacube, lower, upper) - # TODO: HERE, SOMEHOW NEED TO ONLY remove the branches if the values are empty, but only if there are no other possible children left in the tree that we can append and if somehow this happens before and we need to remove, then what do we do?? + # NOTE: need to only remove the branches if the values are empty, + # but only if there are no other possible children left in the tree that + # we can append and if somehow this happens before and we need to remove, then what do we do?? if i == len(right_unsliced_polytopes) - 1: # we have iterated all polytopes and we can now remove the node if we need to if len(values) == 0 and len(node.children) == 0: From 1a3eb9d1adfa7d249bbc356cc0bca69cd96e2bc8 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 14:47:46 +0200 Subject: [PATCH 06/10] update ignore codecov --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codecov.yml b/codecov.yml index 5c74f4da..f75c21fe 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,6 @@ ignore: - "tests" # ignore tests folder + - "**/test*" + - "**/pb2*" + - "**/fdb*" # ignore fdb backend which we can't test in CI yet - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet \ No newline at end of file From 501d03d21a88b778428e8342ac3667e12dc6c76d Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Aug 2024 15:30:56 +0200 Subject: [PATCH 07/10] update docs requirements --- docs/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f17eefa7..710d7792 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ jinja2>=3.1.3 -Markdown<3.2 -# mkdocs>=1.3.0 \ No newline at end of file +Markdown \ No newline at end of file From d39424b4e89ea303ab3cefc453af8f9c9015f4af Mon Sep 17 00:00:00 2001 From: awarde96 Date: Fri, 16 Aug 2024 08:10:13 +0000 Subject: [PATCH 08/10] Refactor options so options config class can now be called by other python modules --- polytope/options.py | 102 +++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/polytope/options.py b/polytope/options.py index 7cce36b9..78c7a20d 100644 --- a/polytope/options.py +++ b/polytope/options.py @@ -6,54 +6,66 @@ from pydantic import ConfigDict +class TransformationConfig(ConfigModel): + model_config = ConfigDict(extra="forbid") + name: str = "" + + +class CyclicConfig(TransformationConfig): + name: Literal["cyclic"] + range: List[float] = [0] + + +class MapperConfig(TransformationConfig): + name: Literal["mapper"] + type: str = "" + resolution: Union[int, List[int]] = 0 + axes: List[str] = [""] + local: Optional[List[float]] = None + + +class ReverseConfig(TransformationConfig): + name: Literal["reverse"] + is_reverse: bool = False + + +class TypeChangeConfig(TransformationConfig): + name: Literal["type_change"] + type: str = "int" + + +class MergeConfig(TransformationConfig): + name: Literal["merge"] + other_axis: str = "" + linkers: List[str] = [""] + + +action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig] + + +class AxisConfig(ConfigModel): + axis_name: str = "" + transformations: list[action_subclasses_union] + + +path_subclasses_union = Union[str, int, float] + + +class GribJumpAxesConfig(ConfigModel): + axis_name: str = "" + values: List[str] = [""] + + +class Config(ConfigModel): + axis_config: List[AxisConfig] = [] + compressed_axes_config: List[str] = [""] + pre_path: Optional[Dict[str, path_subclasses_union]] = {} + alternative_axes: List[GribJumpAxesConfig] = [] + + class PolytopeOptions(ABC): @staticmethod def get_polytope_options(options): - class TransformationConfig(ConfigModel): - model_config = ConfigDict(extra="forbid") - name: str = "" - - class CyclicConfig(TransformationConfig): - name: Literal["cyclic"] - range: List[float] = [0] - - class MapperConfig(TransformationConfig): - name: Literal["mapper"] - type: str = "" - resolution: Union[int, List[int]] = 0 - axes: List[str] = [""] - local: Optional[List[float]] = None - - class ReverseConfig(TransformationConfig): - name: Literal["reverse"] - is_reverse: bool = False - - class TypeChangeConfig(TransformationConfig): - name: Literal["type_change"] - type: str = "int" - - class MergeConfig(TransformationConfig): - name: Literal["merge"] - other_axis: str = "" - linkers: List[str] = [""] - - action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig] - - class AxisConfig(ConfigModel): - axis_name: str = "" - transformations: list[action_subclasses_union] - - path_subclasses_union = Union[str, int, float] - - class GribJumpAxesConfig(ConfigModel): - axis_name: str = "" - values: List[str] = [""] - - class Config(ConfigModel): - axis_config: List[AxisConfig] = [] - compressed_axes_config: List[str] = [""] - pre_path: Optional[Dict[str, path_subclasses_union]] = {} - alternative_axes: List[GribJumpAxesConfig] = [] parser = argparse.ArgumentParser(allow_abbrev=False) conflator = Conflator(app_name="polytope", model=Config, cli=False, argparser=parser, **options) From 5179edff85a5c896dbded1842368ff70ee31d32f Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 23 Aug 2024 14:22:10 +0200 Subject: [PATCH 09/10] clean up codecov and gitignore file --- .gitignore | 1 + codecov.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 5dbb14f0..54ea62f1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ polytope_venv polytope_venv_latest new_updated_numpy_venv newest-polytope-venv +serializedTree diff --git a/codecov.yml b/codecov.yml index f75c21fe..f911473e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,6 @@ +codecov: + branch: develop # set new Default branch + ignore: - "tests" # ignore tests folder - "**/test*" From 2bef13ad1329c6466a66452a8118760e7a4afba6 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 28 Aug 2024 12:08:45 +0200 Subject: [PATCH 10/10] new version --- polytope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/version.py b/polytope/version.py index 68cdeee4..382021f3 100644 --- a/polytope/version.py +++ b/polytope/version.py @@ -1 +1 @@ -__version__ = "1.0.5" +__version__ = "1.0.6"