From ff22599c6c624b136b3c091c59b7948280f75b57 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 8 Jul 2024 15:05:47 +0200 Subject: [PATCH] batch all requests to gj extract into one --- polytope/datacube/backends/fdb.py | 11 ++++++----- tests/test_healpix_nested_grid.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index dae7281a..d6718fb0 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -83,7 +83,8 @@ def get(self, requests: TensorIndexTree): # corresponds to first value in the compressed tuples # TODO: here, loop through the fdb requests and request from gj and directly add to the nodes - + complete_list_complete_uncompressed_requests = [] + complete_fdb_decoding_info = [] for j, compressed_request in enumerate(fdb_requests): # TODO: can we do gj extract outside of this loop? uncompressed_request = {} @@ -99,8 +100,8 @@ def get(self, requests: TensorIndexTree): request_combis = product(*interm_branch_tuple_values) # Need to extract the possible requests and add them to the right nodes - complete_list_complete_uncompressed_requests = [] - complete_fdb_decoding_info = [] + # complete_list_complete_uncompressed_requests = [] + # complete_fdb_decoding_info = [] for combi in request_combis: uncompressed_request = {} for i, key in enumerate(compressed_request[0].keys()): @@ -108,8 +109,8 @@ def get(self, requests: TensorIndexTree): complete_uncompressed_request = (uncompressed_request, compressed_request[1]) complete_list_complete_uncompressed_requests.append(complete_uncompressed_request) complete_fdb_decoding_info.append(fdb_requests_decoding_info[j]) - output_values = self.gj.extract(complete_list_complete_uncompressed_requests) - self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) + output_values = self.gj.extract(complete_list_complete_uncompressed_requests) + self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) def get_fdb_requests( self, requests: TensorIndexTree, fdb_requests=[], fdb_requests_decoding_info=[], leaf_path=None diff --git a/tests/test_healpix_nested_grid.py b/tests/test_healpix_nested_grid.py index c3dc5452..7c5e7433 100644 --- a/tests/test_healpix_nested_grid.py +++ b/tests/test_healpix_nested_grid.py @@ -108,6 +108,7 @@ def setup_method(self, method): @pytest.mark.internet @pytest.mark.fdb + @pytest.mark.skip(reason="different fdb schema for climate dt") def test_healpix_nested_grid(self): import pygribjump as gj