Merge pull request #202 from ecmwf/develop

New Release 1.0.6

.gitignore

@@ -24,3 +24,4 @@ polytope_venv
 polytope_venv_latest
 new_updated_numpy_venv
 newest-polytope-venv
+serializedTree

codecov.yml

@@ -1,3 +1,9 @@
+codecov:
+  branch: develop # set new Default branch
+
 ignore:
   - "tests"  # ignore tests folder
+  - "**/test*"
+  - "**/pb2*"
+  - "**/fdb*" # ignore fdb backend which we can't test in CI yet
   - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet

docs/requirements.txt

@@ -1,3 +1,2 @@
 jinja2>=3.1.3
-Markdown<3.2
-# mkdocs>=1.3.0
+Markdown

polytope/datacube/backends/fdb.py

@@ -78,6 +78,7 @@ def check_branching_axes(self, request):
             self._axes.pop(axis_name, None)
 
     def get(self, requests: TensorIndexTree):
+        requests.pprint()
         if len(requests.children) == 0:
             return requests
         fdb_requests = []

polytope/datacube/tensor_index_tree.py

@@ -81,8 +81,12 @@ def __eq__(self, other):
                     for i in range(len(other.values)):
                         other_val = other.values[i]
                         self_val = self.values[i]
-                        if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol):
-                            return False
+                        if self.axis.can_round:
+                            if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol):
+                                return False
+                        else:
+                            if other_val != self_val:
+                                return False
                     return True
 
     def __lt__(self, other):
@@ -217,7 +221,7 @@ def flatten(self):
     def get_ancestors(self):
         ancestors = []
         current_node = self
-        while current_node.axis != TensorIndexTree.root:
+        while current_node.axis.name != "root":
             ancestors.append(current_node)
             current_node = current_node.parent
         return ancestors[::-1]

polytope/engine/hullslicer.py

@@ -109,9 +109,6 @@ def remap_values(self, ax, value):
         return remapped_val
 
     def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_nodes, slice_axis_idx):
-        if len(values) == 0:
-            node.remove_branch()
-
         for i, value in enumerate(values):
             if i == 0:
                 fvalue = ax.to_float(value)
@@ -140,22 +137,39 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_node
 
     def _build_branch(self, ax, node, datacube, next_nodes):
         if ax.name not in self.compressed_axes:
+            parent_node = node.parent
+            right_unsliced_polytopes = []
             for polytope in node["unsliced_polytopes"]:
                 if ax.name in polytope._axes:
+                    right_unsliced_polytopes.append(polytope)
+            # for polytope in node["unsliced_polytopes"]:
+            for i, polytope in enumerate(right_unsliced_polytopes):
+                node._parent = parent_node
+                # if ax.name in polytope._axes:
+                if True:
                     lower, upper, slice_axis_idx = polytope.extents(ax.name)
                     # here, first check if the axis is an unsliceable axis and directly build node if it is
                     # NOTE: we should have already created the ax_is_unsliceable cache before
                     if self.ax_is_unsliceable[ax.name]:
                         self._build_unsliceable_child(polytope, ax, node, datacube, [lower], next_nodes, slice_axis_idx)
                     else:
                         values = self.find_values_between(polytope, ax, node, datacube, lower, upper)
+                        # NOTE: need to only remove the branches if the values are empty,
+                        # but only if there are no other possible children left in the tree that
+                        # we can append and if somehow this happens before and we need to remove, then what do we do??
+                        if i == len(right_unsliced_polytopes) - 1:
+                            # we have iterated all polytopes and we can now remove the node if we need to
+                            if len(values) == 0 and len(node.children) == 0:
+                                node.remove_branch()
                         self._build_sliceable_child(polytope, ax, node, datacube, values, next_nodes, slice_axis_idx)
         else:
             all_values = []
             all_lowers = []
             first_polytope = False
             first_slice_axis_idx = False
+            parent_node = node.parent
             for polytope in node["unsliced_polytopes"]:
+                node._parent = parent_node
                 if ax.name in polytope._axes:
                     # keep track of the first polytope defined on the given axis
                     if not first_polytope:
@@ -173,6 +187,8 @@ def _build_branch(self, ax, node, datacube, next_nodes):
                     first_polytope, ax, node, datacube, all_lowers, next_nodes, first_slice_axis_idx
                 )
             else:
+                if len(all_values) == 0:
+                    node.remove_branch()
                 self._build_sliceable_child(
                     first_polytope, ax, node, datacube, all_values, next_nodes, first_slice_axis_idx
                 )

polytope/options.py

@@ -6,54 +6,66 @@
 from pydantic import ConfigDict
 
 
+class TransformationConfig(ConfigModel):
+    model_config = ConfigDict(extra="forbid")
+    name: str = ""
+
+
+class CyclicConfig(TransformationConfig):
+    name: Literal["cyclic"]
+    range: List[float] = [0]
+
+
+class MapperConfig(TransformationConfig):
+    name: Literal["mapper"]
+    type: str = ""
+    resolution: Union[int, List[int]] = 0
+    axes: List[str] = [""]
+    local: Optional[List[float]] = None
+
+
+class ReverseConfig(TransformationConfig):
+    name: Literal["reverse"]
+    is_reverse: bool = False
+
+
+class TypeChangeConfig(TransformationConfig):
+    name: Literal["type_change"]
+    type: str = "int"
+
+
+class MergeConfig(TransformationConfig):
+    name: Literal["merge"]
+    other_axis: str = ""
+    linkers: List[str] = [""]
+
+
+action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig]
+
+
+class AxisConfig(ConfigModel):
+    axis_name: str = ""
+    transformations: list[action_subclasses_union]
+
+
+path_subclasses_union = Union[str, int, float]
+
+
+class GribJumpAxesConfig(ConfigModel):
+    axis_name: str = ""
+    values: List[str] = [""]
+
+
+class Config(ConfigModel):
+    axis_config: List[AxisConfig] = []
+    compressed_axes_config: List[str] = [""]
+    pre_path: Optional[Dict[str, path_subclasses_union]] = {}
+    alternative_axes: List[GribJumpAxesConfig] = []
+
+
 class PolytopeOptions(ABC):
     @staticmethod
     def get_polytope_options(options):
-        class TransformationConfig(ConfigModel):
-            model_config = ConfigDict(extra="forbid")
-            name: str = ""
-
-        class CyclicConfig(TransformationConfig):
-            name: Literal["cyclic"]
-            range: List[float] = [0]
-
-        class MapperConfig(TransformationConfig):
-            name: Literal["mapper"]
-            type: str = ""
-            resolution: Union[int, List[int]] = 0
-            axes: List[str] = [""]
-            local: Optional[List[float]] = None
-
-        class ReverseConfig(TransformationConfig):
-            name: Literal["reverse"]
-            is_reverse: bool = False
-
-        class TypeChangeConfig(TransformationConfig):
-            name: Literal["type_change"]
-            type: str = "int"
-
-        class MergeConfig(TransformationConfig):
-            name: Literal["merge"]
-            other_axis: str = ""
-            linkers: List[str] = [""]
-
-        action_subclasses_union = Union[CyclicConfig, MapperConfig, ReverseConfig, TypeChangeConfig, MergeConfig]
-
-        class AxisConfig(ConfigModel):
-            axis_name: str = ""
-            transformations: list[action_subclasses_union]
-
-        path_subclasses_union = Union[str, int, float]
-
-        class GribJumpAxesConfig(ConfigModel):
-            axis_name: str = ""
-            values: List[str] = [""]
-
-        class Config(ConfigModel):
-            axis_config: List[AxisConfig] = []
-            compressed_axes_config: List[str] = [""]
-            pre_path: Optional[Dict[str, path_subclasses_union]] = {}
-            alternative_axes: List[GribJumpAxesConfig] = []
 
         parser = argparse.ArgumentParser(allow_abbrev=False)
         conflator = Conflator(app_name="polytope", model=Config, cli=False, argparser=parser, **options)

polytope/version.py

@@ -1 +1 @@
-__version__ = "1.0.5"
+__version__ = "1.0.6"

tests/test_slice_date_range_fdb.py

@@ -73,6 +73,96 @@ def test_fdb_datacube(self):
         for i in range(len(result.leaves)):
             assert len(result.leaves[i].result) == 3
 
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_non_existing_last(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 3
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 3
+
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_non_existing_first(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230625T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 3
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 3
+
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_completely_non_existing(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230626T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 1
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 0
+
     @pytest.mark.fdb
     def test_fdb_datacube_disk(self):
         import pygribjump as gj