-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
towards v1.7: new CMS datasets, CLIC hit-based datasets, TF backward-…
…compat optimizations (#285) * training with cms 1.7.0 * fix postprocessing for new uproot * track memory usage of pandora * readd dim decrease options * optimizer save/restore * hypertune * update track feats * add pytorch training on clic hits --------- Co-authored-by: Joosep Pata <joosep.pata@kbfi.ee>
- Loading branch information
Showing
99 changed files
with
2,418 additions
and
1,229 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| from pathlib import Path | ||
|
|
||
| import tensorflow as tf | ||
| from utils_edm import ( | ||
| X_FEATURES_CL, | ||
| X_FEATURES_TRK, | ||
| Y_FEATURES, | ||
| generate_examples, | ||
| split_sample, | ||
| ) | ||
|
|
||
| import tensorflow_datasets as tfds | ||
| import numpy as np | ||
|
|
||
| _DESCRIPTION = """ | ||
| CLIC EDM4HEP dataset with single gamma particle gun | ||
| - X: reconstructed tracks and clusters, variable number N per event | ||
| - ygen: stable generator particles, zero-padded to N per event | ||
| - ycand: baseline particle flow particles, zero-padded to N per event | ||
| """ | ||
|
|
||
| _CITATION = """ | ||
| Pata, Joosep, Wulff, Eric, Duarte, Javier, Mokhtar, Farouk, Zhang, Mengke, Girone, Maria, & Southwick, David. (2023). | ||
| Simulated datasets for detector and particle flow reconstruction: CLIC detector (1.1) [Data set]. | ||
| Zenodo. https://doi.org/10.5281/zenodo.8260741 | ||
| """ | ||
|
|
||
|
|
||
| class ClicEdmSingleGammaPf(tfds.core.GeneratorBasedBuilder): | ||
| VERSION = tfds.core.Version("1.5.0") | ||
| RELEASE_NOTES = { | ||
| "1.5.0": "Regenerate with ARRAY_RECORD", | ||
| } | ||
| MANUAL_DOWNLOAD_INSTRUCTIONS = """ | ||
| For the raw input files in ROOT EDM4HEP format, please see the citation above. | ||
| The processed tensorflow_dataset can also be downloaded from: | ||
| rsync -r --progress lxplus.cern.ch:/eos/user/j/jpata/mlpf/clic_edm4hep/ ./ | ||
| """ | ||
|
|
||
| def __init__(self, *args, **kwargs): | ||
| kwargs["file_format"] = tfds.core.FileFormat.ARRAY_RECORD | ||
| super(ClicEdmSingleGammaPf, self).__init__(*args, **kwargs) | ||
|
|
||
| def _info(self) -> tfds.core.DatasetInfo: | ||
| """Returns the dataset metadata.""" | ||
| return tfds.core.DatasetInfo( | ||
| builder=self, | ||
| description=_DESCRIPTION, | ||
| features=tfds.features.FeaturesDict( | ||
| { | ||
| "X": tfds.features.Tensor( | ||
| shape=( | ||
| None, | ||
| max(len(X_FEATURES_TRK), len(X_FEATURES_CL)), | ||
| ), | ||
| dtype=tf.float32, | ||
| ), | ||
| "ygen": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| "ycand": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| } | ||
| ), | ||
| supervised_keys=None, | ||
| homepage="", | ||
| citation=_CITATION, | ||
| metadata=tfds.core.MetadataDict( | ||
| x_features_track=X_FEATURES_TRK, | ||
| x_features_cluster=X_FEATURES_CL, | ||
| y_features=Y_FEATURES, | ||
| ), | ||
| ) | ||
|
|
||
| def _split_generators(self, dl_manager: tfds.download.DownloadManager): | ||
| path = dl_manager.manual_dir | ||
| return split_sample(Path(path / "gamma/")) | ||
|
|
||
| def _generate_examples(self, files): | ||
| return generate_examples(files) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| from pathlib import Path | ||
|
|
||
| import tensorflow as tf | ||
| from utils_edm import ( | ||
| X_FEATURES_CL, | ||
| X_FEATURES_TRK, | ||
| Y_FEATURES, | ||
| generate_examples, | ||
| split_sample, | ||
| ) | ||
|
|
||
| import tensorflow_datasets as tfds | ||
| import numpy as np | ||
|
|
||
| _DESCRIPTION = """ | ||
| CLIC EDM4HEP dataset with single kaon0L particle gun | ||
| - X: reconstructed tracks and clusters, variable number N per event | ||
| - ygen: stable generator particles, zero-padded to N per event | ||
| - ycand: baseline particle flow particles, zero-padded to N per event | ||
| """ | ||
|
|
||
| _CITATION = """ | ||
| Pata, Joosep, Wulff, Eric, Duarte, Javier, Mokhtar, Farouk, Zhang, Mengke, Girone, Maria, & Southwick, David. (2023). | ||
| Simulated datasets for detector and particle flow reconstruction: CLIC detector (1.1) [Data set]. | ||
| Zenodo. https://doi.org/10.5281/zenodo.8260741 | ||
| """ | ||
|
|
||
|
|
||
| class ClicEdmSingleKaon0lPf(tfds.core.GeneratorBasedBuilder): | ||
| VERSION = tfds.core.Version("1.5.0") | ||
| RELEASE_NOTES = { | ||
| "1.5.0": "Regenerate with ARRAY_RECORD", | ||
| } | ||
| MANUAL_DOWNLOAD_INSTRUCTIONS = """ | ||
| For the raw input files in ROOT EDM4HEP format, please see the citation above. | ||
| The processed tensorflow_dataset can also be downloaded from: | ||
| rsync -r --progress lxplus.cern.ch:/eos/user/j/jpata/mlpf/clic_edm4hep/ ./ | ||
| """ | ||
|
|
||
| def __init__(self, *args, **kwargs): | ||
| kwargs["file_format"] = tfds.core.FileFormat.ARRAY_RECORD | ||
| super(ClicEdmSingleKaon0lPf, self).__init__(*args, **kwargs) | ||
|
|
||
| def _info(self) -> tfds.core.DatasetInfo: | ||
| """Returns the dataset metadata.""" | ||
| return tfds.core.DatasetInfo( | ||
| builder=self, | ||
| description=_DESCRIPTION, | ||
| features=tfds.features.FeaturesDict( | ||
| { | ||
| "X": tfds.features.Tensor( | ||
| shape=( | ||
| None, | ||
| max(len(X_FEATURES_TRK), len(X_FEATURES_CL)), | ||
| ), | ||
| dtype=tf.float32, | ||
| ), | ||
| "ygen": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| "ycand": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| } | ||
| ), | ||
| supervised_keys=None, | ||
| homepage="", | ||
| citation=_CITATION, | ||
| metadata=tfds.core.MetadataDict( | ||
| x_features_track=X_FEATURES_TRK, | ||
| x_features_cluster=X_FEATURES_CL, | ||
| y_features=Y_FEATURES, | ||
| ), | ||
| ) | ||
|
|
||
| def _split_generators(self, dl_manager: tfds.download.DownloadManager): | ||
| path = dl_manager.manual_dir | ||
| return split_sample(Path(path / "kaon0L/")) | ||
|
|
||
| def _generate_examples(self, files): | ||
| return generate_examples(files) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| from pathlib import Path | ||
|
|
||
| import tensorflow as tf | ||
| from utils_edm import ( | ||
| X_FEATURES_CL, | ||
| X_FEATURES_TRK, | ||
| Y_FEATURES, | ||
| generate_examples, | ||
| split_sample_several, | ||
| ) | ||
|
|
||
| import tensorflow_datasets as tfds | ||
| import numpy as np | ||
|
|
||
| _DESCRIPTION = """ | ||
| CLIC EDM4HEP dataset with single-pion particle gun | ||
| - X: reconstructed tracks and clusters, variable number N per event | ||
| - ygen: stable generator particles, zero-padded to N per event | ||
| - ycand: baseline particle flow particles, zero-padded to N per event | ||
| """ | ||
|
|
||
| _CITATION = """ | ||
| Pata, Joosep, Wulff, Eric, Duarte, Javier, Mokhtar, Farouk, Zhang, Mengke, Girone, Maria, & Southwick, David. (2023). | ||
| Simulated datasets for detector and particle flow reconstruction: CLIC detector (1.1) [Data set]. | ||
| Zenodo. https://doi.org/10.5281/zenodo.8260741 | ||
| """ | ||
|
|
||
|
|
||
| class ClicEdmSinglePiPf(tfds.core.GeneratorBasedBuilder): | ||
| VERSION = tfds.core.Version("1.5.0") | ||
| RELEASE_NOTES = { | ||
| "1.5.0": "Regenerate with ARRAY_RECORD", | ||
| } | ||
| MANUAL_DOWNLOAD_INSTRUCTIONS = """ | ||
| For the raw input files in ROOT EDM4HEP format, please see the citation above. | ||
| The processed tensorflow_dataset can also be downloaded from: | ||
| rsync -r --progress lxplus.cern.ch:/eos/user/j/jpata/mlpf/clic_edm4hep/ ./ | ||
| """ | ||
|
|
||
| def __init__(self, *args, **kwargs): | ||
| kwargs["file_format"] = tfds.core.FileFormat.ARRAY_RECORD | ||
| super(ClicEdmSinglePiPf, self).__init__(*args, **kwargs) | ||
|
|
||
| def _info(self) -> tfds.core.DatasetInfo: | ||
| """Returns the dataset metadata.""" | ||
| return tfds.core.DatasetInfo( | ||
| builder=self, | ||
| description=_DESCRIPTION, | ||
| features=tfds.features.FeaturesDict( | ||
| { | ||
| "X": tfds.features.Tensor( | ||
| shape=( | ||
| None, | ||
| max(len(X_FEATURES_TRK), len(X_FEATURES_CL)), | ||
| ), | ||
| dtype=tf.float32, | ||
| ), | ||
| "ygen": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| "ycand": tfds.features.Tensor(shape=(None, len(Y_FEATURES)), dtype=np.float32), | ||
| } | ||
| ), | ||
| supervised_keys=None, | ||
| homepage="", | ||
| citation=_CITATION, | ||
| metadata=tfds.core.MetadataDict( | ||
| x_features_track=X_FEATURES_TRK, | ||
| x_features_cluster=X_FEATURES_CL, | ||
| y_features=Y_FEATURES, | ||
| ), | ||
| ) | ||
|
|
||
| def _split_generators(self, dl_manager: tfds.download.DownloadManager): | ||
| path = dl_manager.manual_dir | ||
| return split_sample_several([Path(path / "pi-/"), Path(path / "pi+/")]) | ||
|
|
||
| def _generate_examples(self, files): | ||
| return generate_examples(files) |
Oops, something went wrong.