diff --git a/simtools/applications/production.py b/simtools/applications/production.py index e107d38ca..2f1126095 100644 --- a/simtools/applications/production.py +++ b/simtools/applications/production.py @@ -23,7 +23,7 @@ Command line arguments ---------------------- - productionconfig (str, required) + simulation_configuration (str, required) Path to the simulation configuration file. primary (str) Name of the primary to be selected from the configuration file. In case it \ @@ -34,7 +34,7 @@ simulate (perform simulations), file_list (print list of output files) inspect (plot sim_telarray histograms for quick inspection) - resources (print quicklook into used computational resources) + resources (print quick look into used computational resources) array_only (activation mode) Simulates only array detector (no showers). showers_only (activation mode) @@ -66,7 +66,7 @@ .. code-block:: console - simtools-production --task simulate --productionconfig prod_config_test.yml \ + simtools-production --task simulate --simulation_configuration prod_config_test.yml \ --test --showers_only --submit_command local The output is saved in simtools-output/test-production. @@ -83,8 +83,6 @@ from copy import copy from pathlib import Path -from astropy.io.misc import yaml - import simtools.utils.general as gen from simtools.configuration import configurator from simtools.simulator import Simulator @@ -108,7 +106,7 @@ def _parse(description=None): config = configurator.Configurator(description=description) config.parser.add_argument( - "--productionconfig", + "--simulation_configuration", help="Simulation configuration file", type=str, required=True, @@ -193,8 +191,7 @@ def _process_simulation_config_file(config_file, primary_config, logger): """ try: - with open(config_file, encoding="utf-8") as file: - config_data = yaml.load(file) + config_data = gen.collect_data_from_file_or_dict(file_name=config_file, in_dict=None) except FileNotFoundError: logger.error(f"Error loading simulation configuration file from {config_file}") raise @@ -213,24 +210,24 @@ def _process_simulation_config_file(config_file, primary_config, logger): config_showers[primary] = copy(this_default.pop("showers", {})) config_arrays[primary] = copy(this_default.pop("array", {})) - # Grabbing common entries for showers and array + # common entries for showers and array for key, value in primary_data.items(): if key in ["showers", "array"]: continue config_showers[primary][key] = value config_arrays[primary][key] = value - # Grabbing showers entries + # shower entries for key, value in primary_data.get("showers", {}).items(): config_showers[primary][key] = value config_showers[primary]["primary"] = primary - # Grabbing array entries + # array entries for key, value in primary_data.get("array", {}).items(): config_arrays[primary][key] = value config_arrays[primary]["primary"] = primary - # Filling in the remaining default keys + # remaining default keys for key, value in this_default.items(): config_showers[primary][key] = value config_arrays[primary][key] = value @@ -244,7 +241,7 @@ def main(): logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) label, shower_configs, array_configs = _process_simulation_config_file( - args_dict["productionconfig"], args_dict["primary"], logger + args_dict["simulation_configuration"], args_dict["primary"], logger ) if args_dict["label"] is None: args_dict["label"] = label diff --git a/simtools/simulator.py b/simtools/simulator.py index e19422d1a..13f7b6b4e 100644 --- a/simtools/simulator.py +++ b/simtools/simulator.py @@ -501,7 +501,7 @@ def _get_runs_and_files_to_submit(self, input_file_list=None): Returns ------- runs_and_files: dict - dictionary with runnumber as key and (if available) simulation + dictionary with run number as key and (if available) simulation file name as value """ diff --git a/tests/integration_tests/config/production_array_only.yml b/tests/integration_tests/config/production_array_only.yml index e1862cb39..411484655 100644 --- a/tests/integration_tests/config/production_array_only.yml +++ b/tests/integration_tests/config/production_array_only.yml @@ -2,7 +2,7 @@ CTA_SIMPIPE: APPLICATION: simtools-production TEST_NAME: array_only CONFIGURATION: - PRODUCTIONCONFIG: ./tests/resources/prod_config_test.yml + SIMULATION_CONFIG: ./tests/resources/prod_config_test.yml TASK: simulate ARRAY_ONLY: true SUBMIT_COMMAND: local diff --git a/tests/integration_tests/config/production_showers_only.yml b/tests/integration_tests/config/production_showers_only.yml index 834e2c670..247a72a8f 100644 --- a/tests/integration_tests/config/production_showers_only.yml +++ b/tests/integration_tests/config/production_showers_only.yml @@ -2,13 +2,14 @@ CTA_SIMPIPE: APPLICATION: simtools-production TEST_NAME: showers_only CONFIGURATION: - PRODUCTIONCONFIG: ./tests/resources/prod_config_test.yml + SIMULATION_CONFIGURATION: ./tests/resources/prod_config_test.yml MODEL_VERSION: prod6 TASK: simulate SHOWERS_ONLY: true SUBMIT_COMMAND: local DATA_DIRECTORY: simtools-output OUTPUT_PATH: simtools-output + LOG_LEVEL: debug INTEGRATION_TESTS: - OUTPUT_FILE: | corsika-data/North/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_North_ diff --git a/tests/integration_tests/config/sim_showers_for_trigger_rates_run.yml b/tests/integration_tests/config/sim_showers_for_trigger_rates_run.yml index ed3b24484..cc1b6fa94 100644 --- a/tests/integration_tests/config/sim_showers_for_trigger_rates_run.yml +++ b/tests/integration_tests/config/sim_showers_for_trigger_rates_run.yml @@ -7,8 +7,10 @@ CTA_SIMPIPE: PRIMARY: proton MODEL_VERSION: prod6 NRUNS: 2 - NEVENTS: 10000 - TEST: true + NEVENTS: 2 SUBMIT_COMMAND: local DATA_DIRECTORY: simtools-output OUTPUT_PATH: simtools-output + INTEGRATION_TESTS: + - OUTPUT_FILE: | + sim_showers_for_trigger_rates/corsika-data/North/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_North_4LST_sim_showers_for_trigger_rates.zst diff --git a/tests/integration_tests/config/sim_showers_for_trigger_rates_run_South.yml b/tests/integration_tests/config/sim_showers_for_trigger_rates_run_South.yml new file mode 100644 index 000000000..71eac0513 --- /dev/null +++ b/tests/integration_tests/config/sim_showers_for_trigger_rates_run_South.yml @@ -0,0 +1,16 @@ +CTA_SIMPIPE: + APPLICATION: simtools-sim-showers-for-trigger-rates + TEST_NAME: run_south + CONFIGURATION: + ARRAY: 4MST + SITE: South + PRIMARY: proton + MODEL_VERSION: prod6 + NRUNS: 2 + NEVENTS: 2 + SUBMIT_COMMAND: local + DATA_DIRECTORY: simtools-output + OUTPUT_PATH: simtools-output + INTEGRATION_TESTS: + - OUTPUT_FILE: | + sim_showers_for_trigger_rates/corsika-data/South/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_South_4MST_sim_showers_for_trigger_rates.zst diff --git a/tests/integration_tests/config/simulate_prod_gamma_20_deg_North.yml b/tests/integration_tests/config/simulate_prod_gamma_20_deg_North.yml index e79b16a89..bd5afe82b 100644 --- a/tests/integration_tests/config/simulate_prod_gamma_20_deg_North.yml +++ b/tests/integration_tests/config/simulate_prod_gamma_20_deg_North.yml @@ -13,3 +13,10 @@ CTA_SIMPIPE: NSHOW: 5 DATA_DIRECTORY: simtools-output OUTPUT_PATH: simtools-output + INTEGRATION_TESTS: + - OUTPUT_FILE: | + test-production-North/simtel-data/North/gamma/data/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production-North.simtel.zst + - OUTPUT_FILE: | + test-production-North/simtel-data/North/gamma/log/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production-North.log.gz + - OUTPUT_FILE: | + test-production-North/simtel-data/North/gamma/log/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production-North.hdata.zst diff --git a/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml b/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml index 61df5763f..289fc2a2f 100644 --- a/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml +++ b/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml @@ -14,3 +14,8 @@ CTA_SIMPIPE: DATA_DIRECTORY: simtools-output OUTPUT_PATH: simtools-output PACK_FOR_GRID_REGISTER: true + INTEGRATION_TESTS: + - OUTPUT_FILE: | + directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production-North.log_hist.tar.gz + - OUTPUT_FILE: | + directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production-North.simtel.zst diff --git a/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid_south.yml b/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid_south.yml new file mode 100644 index 000000000..67505aae3 --- /dev/null +++ b/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid_south.yml @@ -0,0 +1,21 @@ +CTA_SIMPIPE: + APPLICATION: simtools-simulate-prod + TEST_NAME: gamma_20_deg_pack_for_grid_south + CONFIGURATION: + PRODUCTION_CONFIG: ./tests/resources/prod_multi_config_test_South.yml + MODEL_VERSION: prod6 + SITE: South + PRIMARY: gamma + AZIMUTH_ANGLE: North + ZENITH_ANGLE: 20 + START_RUN: 0 + RUN: 2 + NSHOW: 5 + DATA_DIRECTORY: simtools-output + OUTPUT_PATH: simtools-output + PACK_FOR_GRID_REGISTER: true + INTEGRATION_TESTS: + - OUTPUT_FILE: | + directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_South_TestLayout_test-production-South.log_hist.tar.gz + - OUTPUT_FILE: | + directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_South_TestLayout_test-production-South.simtel.zst diff --git a/tests/integration_tests/test_applications_from_config.py b/tests/integration_tests/test_applications_from_config.py index cb13c51c0..49aa5222b 100644 --- a/tests/integration_tests/test_applications_from_config.py +++ b/tests/integration_tests/test_applications_from_config.py @@ -248,7 +248,7 @@ def validate_application_output(config): # First check if the output is in the data directory (simtel_array related), # Then check if the file is in the output directory (remaining tools). logger.info(f"PATH {config['CONFIGURATION']['OUTPUT_PATH']}") - logger.info(f"File{integration_test['OUTPUT_FILE']}") + logger.info(f"File {integration_test['OUTPUT_FILE']}") try: assert ( Path(config["CONFIGURATION"]["DATA_DIRECTORY"])