Feature/fix undeterministic bug

polytope/datacube/backends/fdb.py

@@ -78,6 +78,7 @@ def check_branching_axes(self, request):
             self._axes.pop(axis_name, None)
 
     def get(self, requests: TensorIndexTree):
+        requests.pprint()
         if len(requests.children) == 0:
             return requests
         fdb_requests = []

polytope/datacube/tensor_index_tree.py

@@ -81,8 +81,12 @@ def __eq__(self, other):
                     for i in range(len(other.values)):
                         other_val = other.values[i]
                         self_val = self.values[i]
-                        if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol):
-                            return False
+                        if self.axis.can_round:
+                            if abs(other_val - self_val) > 2 * max(other.axis.tol, self.axis.tol):
+                                return False
+                        else:
+                            if other_val != self_val:
+                                return False
                     return True
 
     def __lt__(self, other):
@@ -217,7 +221,7 @@ def flatten(self):
     def get_ancestors(self):
         ancestors = []
         current_node = self
-        while current_node.axis != TensorIndexTree.root:
+        while current_node.axis.name != "root":
             ancestors.append(current_node)
             current_node = current_node.parent
         return ancestors[::-1]

polytope/engine/hullslicer.py

@@ -109,9 +109,6 @@ def remap_values(self, ax, value):
         return remapped_val
 
     def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_nodes, slice_axis_idx):
-        if len(values) == 0:
-            node.remove_branch()
-
         for i, value in enumerate(values):
             if i == 0:
                 fvalue = ax.to_float(value)
@@ -140,22 +137,39 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, values, next_node
 
     def _build_branch(self, ax, node, datacube, next_nodes):
         if ax.name not in self.compressed_axes:
+            parent_node = node.parent
+            right_unsliced_polytopes = []
             for polytope in node["unsliced_polytopes"]:
                 if ax.name in polytope._axes:
+                    right_unsliced_polytopes.append(polytope)
+            # for polytope in node["unsliced_polytopes"]:
+            for i, polytope in enumerate(right_unsliced_polytopes):
+                node._parent = parent_node
+                # if ax.name in polytope._axes:
+                if True:
                     lower, upper, slice_axis_idx = polytope.extents(ax.name)
                     # here, first check if the axis is an unsliceable axis and directly build node if it is
                     # NOTE: we should have already created the ax_is_unsliceable cache before
                     if self.ax_is_unsliceable[ax.name]:
                         self._build_unsliceable_child(polytope, ax, node, datacube, [lower], next_nodes, slice_axis_idx)
                     else:
                         values = self.find_values_between(polytope, ax, node, datacube, lower, upper)
+                        # NOTE: need to only remove the branches if the values are empty,
+                        # but only if there are no other possible children left in the tree that
+                        # we can append and if somehow this happens before and we need to remove, then what do we do??
+                        if i == len(right_unsliced_polytopes) - 1:
+                            # we have iterated all polytopes and we can now remove the node if we need to
+                            if len(values) == 0 and len(node.children) == 0:
+                                node.remove_branch()
                         self._build_sliceable_child(polytope, ax, node, datacube, values, next_nodes, slice_axis_idx)
         else:
             all_values = []
             all_lowers = []
             first_polytope = False
             first_slice_axis_idx = False
+            parent_node = node.parent
             for polytope in node["unsliced_polytopes"]:
+                node._parent = parent_node
                 if ax.name in polytope._axes:
                     # keep track of the first polytope defined on the given axis
                     if not first_polytope:
@@ -173,6 +187,8 @@ def _build_branch(self, ax, node, datacube, next_nodes):
                     first_polytope, ax, node, datacube, all_lowers, next_nodes, first_slice_axis_idx
                 )
             else:
+                if len(all_values) == 0:
+                    node.remove_branch()
                 self._build_sliceable_child(
                     first_polytope, ax, node, datacube, all_values, next_nodes, first_slice_axis_idx
                 )

tests/test_slice_date_range_fdb.py

@@ -73,6 +73,96 @@ def test_fdb_datacube(self):
         for i in range(len(result.leaves)):
             assert len(result.leaves[i].result) == 3
 
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_non_existing_last(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 3
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 3
+
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_non_existing_first(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230625T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 3
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 3
+
+    @pytest.mark.fdb
+    def test_fdb_datacube_select_completely_non_existing(self):
+        import pygribjump as gj
+
+        request = Request(
+            Select("step", [0]),
+            Select("levtype", ["sfc"]),
+            Select("date", [pd.Timestamp("20230624T120000"), pd.Timestamp("20230626T120000")]),
+            Select("domain", ["g"]),
+            Select("expver", ["0001"]),
+            Select("param", ["167"]),
+            Select("class", ["od"]),
+            Select("stream", ["oper"]),
+            Select("type", ["an"]),
+            Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]),
+        )
+
+        self.fdbdatacube = gj.GribJump()
+        self.slicer = HullSlicer()
+        self.API = Polytope(
+            datacube=self.fdbdatacube,
+            engine=self.slicer,
+            options=self.options,
+        )
+        result = self.API.retrieve(request)
+        result.pprint()
+        assert len(result.leaves) == 1
+        for i in range(len(result.leaves)):
+            assert len(result.leaves[i].result) == 0
+
     @pytest.mark.fdb
     def test_fdb_datacube_disk(self):
         import pygribjump as gj