From 926b984005cc6e1bf1e72cc056e1f2437e7bab23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20S=C3=B8gaard?= Date: Thu, 26 Jan 2023 19:45:36 +0100 Subject: [PATCH 1/2] Remove paths relative to GRAPHNET_ROOT_DIR --- src/graphnet/data/dataset.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/graphnet/data/dataset.py b/src/graphnet/data/dataset.py index 26f7006ed..d07585223 100644 --- a/src/graphnet/data/dataset.py +++ b/src/graphnet/data/dataset.py @@ -9,6 +9,7 @@ from torch.utils.data import ConcatDataset from torch_geometric.data import Data +from graphnet.constants import GRAPHNET_ROOT_DIR from graphnet.utilities.config import ( Configurable, DatasetConfig, @@ -105,6 +106,21 @@ def _construct_dataset_from_list_of_strings( return cls.concatenate(datasets) + @classmethod + def _resolve_graphnet_paths( + cls, path: Union[str, List[str]] + ) -> Union[str, List[str]]: + if isinstance(path, list): + return [cast(str, cls._resolve_graphnet_paths(p)) for p in path] + + assert isinstance(path, str) + return ( + path.replace("$graphnet", GRAPHNET_ROOT_DIR) + .replace("$GRAPHNET", GRAPHNET_ROOT_DIR) + .replace("${graphnet}", GRAPHNET_ROOT_DIR) + .replace("${GRAPHNET}", GRAPHNET_ROOT_DIR) + ) + @save_dataset_config def __init__( self, @@ -177,6 +193,9 @@ def __init__( assert isinstance(features, (list, tuple)) assert isinstance(truth, (list, tuple)) + # Resolve reference to `$GRAPHNET` in path(s) + path = self._resolve_graphnet_paths(path) + # Member variable(s) self._path = path self._selection = None From a26d5d740619aaa133a41d1edb6eca021c1287c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20S=C3=B8gaard?= Date: Thu, 26 Jan 2023 19:46:02 +0100 Subject: [PATCH 2/2] Update paths --- configs/datasets/test_data_sqlite.yml | 2 +- configs/datasets/training_example_data_parquet.yml | 2 +- configs/datasets/training_example_data_sqlite.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/datasets/test_data_sqlite.yml b/configs/datasets/test_data_sqlite.yml index ae4990f65..9ea481d74 100644 --- a/configs/datasets/test_data_sqlite.yml +++ b/configs/datasets/test_data_sqlite.yml @@ -1,4 +1,4 @@ -path: /groups/icecube/asogaard/work/development/graphnet/data/tests/sqlite/oscNext_genie_level7_v02/oscNext_genie_level7_v02_first_5_frames.db +path: $GRAPHNET/data/tests/sqlite/oscNext_genie_level7_v02/oscNext_genie_level7_v02_first_5_frames.db pulsemaps: - SRTInIcePulses features: diff --git a/configs/datasets/training_example_data_parquet.yml b/configs/datasets/training_example_data_parquet.yml index 46e9fe167..9849a5d5e 100644 --- a/configs/datasets/training_example_data_parquet.yml +++ b/configs/datasets/training_example_data_parquet.yml @@ -1,4 +1,4 @@ -path: /groups/icecube/asogaard/work/development/graphnet/data/examples/parquet/prometheus/prometheus-events.parquet +path: $GRAPHNET/data/examples/parquet/prometheus/prometheus-events.parquet pulsemaps: - total features: diff --git a/configs/datasets/training_example_data_sqlite.yml b/configs/datasets/training_example_data_sqlite.yml index 2d99c8f56..72d1e9c64 100644 --- a/configs/datasets/training_example_data_sqlite.yml +++ b/configs/datasets/training_example_data_sqlite.yml @@ -1,4 +1,4 @@ -path: /groups/icecube/asogaard/work/development/graphnet/data/examples/sqlite/prometheus/prometheus-events.db +path: $GRAPHNET/data/examples/sqlite/prometheus/prometheus-events.db pulsemaps: - total features: