Simulate prod test

simtools/applications/production.py

@@ -23,7 +23,7 @@
 
     Command line arguments
     ----------------------
-    productionconfig (str, required)
+    simulation_configuration (str, required)
         Path to the simulation configuration file.
     primary (str)
         Name of the primary to be selected from the configuration file. In case it \
@@ -34,7 +34,7 @@
             simulate (perform simulations),
             file_list (print list of output files)
             inspect (plot sim_telarray histograms for quick inspection)
-            resources (print quicklook into used computational resources)
+            resources (print quick look into used computational resources)
     array_only (activation mode)
         Simulates only array detector (no showers).
     showers_only (activation mode)
@@ -66,7 +66,7 @@
 
     .. code-block:: console
 
-        simtools-production --task simulate --productionconfig prod_config_test.yml \
+        simtools-production --task simulate --simulation_configuration prod_config_test.yml \
         --test --showers_only --submit_command local
 
     The output is saved in simtools-output/test-production.
@@ -83,8 +83,6 @@
 from copy import copy
 from pathlib import Path
 
-from astropy.io.misc import yaml
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.simulator import Simulator
@@ -108,7 +106,7 @@ def _parse(description=None):
 
     config = configurator.Configurator(description=description)
     config.parser.add_argument(
-        "--productionconfig",
+        "--simulation_configuration",
         help="Simulation configuration file",
         type=str,
         required=True,
@@ -193,8 +191,7 @@ def _process_simulation_config_file(config_file, primary_config, logger):
     """
 
     try:
-        with open(config_file, encoding="utf-8") as file:
-            config_data = yaml.load(file)
+        config_data = gen.collect_data_from_file_or_dict(file_name=config_file, in_dict=None)
     except FileNotFoundError:
         logger.error(f"Error loading simulation configuration file from {config_file}")
         raise
@@ -213,24 +210,24 @@ def _process_simulation_config_file(config_file, primary_config, logger):
         config_showers[primary] = copy(this_default.pop("showers", {}))
         config_arrays[primary] = copy(this_default.pop("array", {}))
 
-        # Grabbing common entries for showers and array
+        # common entries for showers and array
         for key, value in primary_data.items():
             if key in ["showers", "array"]:
                 continue
             config_showers[primary][key] = value
             config_arrays[primary][key] = value
 
-        # Grabbing showers entries
+        # shower entries
         for key, value in primary_data.get("showers", {}).items():
             config_showers[primary][key] = value
         config_showers[primary]["primary"] = primary
 
-        # Grabbing array entries
+        # array entries
         for key, value in primary_data.get("array", {}).items():
             config_arrays[primary][key] = value
         config_arrays[primary]["primary"] = primary
 
-        # Filling in the remaining default keys
+        # remaining default keys
         for key, value in this_default.items():
             config_showers[primary][key] = value
             config_arrays[primary][key] = value
@@ -244,7 +241,7 @@ def main():
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     label, shower_configs, array_configs = _process_simulation_config_file(
-        args_dict["productionconfig"], args_dict["primary"], logger
+        args_dict["simulation_configuration"], args_dict["primary"], logger
     )
     if args_dict["label"] is None:
         args_dict["label"] = label

simtools/simulator.py

@@ -501,7 +501,7 @@ def _get_runs_and_files_to_submit(self, input_file_list=None):
         Returns
         -------
         runs_and_files: dict
-            dictionary with runnumber as key and (if available) simulation
+            dictionary with run number as key and (if available) simulation
             file name as value
 
         """

tests/integration_tests/config/production_array_only.yml

@@ -2,7 +2,7 @@ CTA_SIMPIPE:
   APPLICATION: simtools-production
   TEST_NAME: array_only
   CONFIGURATION:
-    PRODUCTIONCONFIG: ./tests/resources/prod_config_test.yml
+    SIMULATION_CONFIG: ./tests/resources/prod_config_test.yml
     TASK: simulate
     ARRAY_ONLY: true
     SUBMIT_COMMAND: local

tests/integration_tests/config/production_showers_only.yml

@@ -2,13 +2,14 @@ CTA_SIMPIPE:
   APPLICATION: simtools-production
   TEST_NAME: showers_only
   CONFIGURATION:
-    PRODUCTIONCONFIG: ./tests/resources/prod_config_test.yml
+    SIMULATION_CONFIGURATION: ./tests/resources/prod_config_test.yml
     MODEL_VERSION: prod6
     TASK: simulate
     SHOWERS_ONLY: true
     SUBMIT_COMMAND: local
     DATA_DIRECTORY: simtools-output
     OUTPUT_PATH: simtools-output
+    LOG_LEVEL: debug
   INTEGRATION_TESTS:
     - OUTPUT_FILE: |
         corsika-data/North/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_North_

tests/integration_tests/config/sim_showers_for_trigger_rates_run.yml

@@ -7,8 +7,10 @@ CTA_SIMPIPE:
     PRIMARY: proton
     MODEL_VERSION: prod6
     NRUNS: 2
-    NEVENTS: 10000
-    TEST: true
+    NEVENTS: 2
     SUBMIT_COMMAND: local
     DATA_DIRECTORY: simtools-output
     OUTPUT_PATH: simtools-output
+  INTEGRATION_TESTS:
+    - OUTPUT_FILE: |
+        sim_showers_for_trigger_rates/corsika-data/North/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_North_4LST_sim_showers_for_trigger_rates.zst

tests/integration_tests/config/sim_showers_for_trigger_rates_run_South.yml

@@ -0,0 +1,16 @@
+CTA_SIMPIPE:
+  APPLICATION: simtools-sim-showers-for-trigger-rates
+  TEST_NAME: run_south
+  CONFIGURATION:
+    ARRAY: 4MST
+    SITE: South
+    PRIMARY: proton
+    MODEL_VERSION: prod6
+    NRUNS: 2
+    NEVENTS: 2
+    SUBMIT_COMMAND: local
+    DATA_DIRECTORY: simtools-output
+    OUTPUT_PATH: simtools-output
+  INTEGRATION_TESTS:
+    - OUTPUT_FILE: |
+        sim_showers_for_trigger_rates/corsika-data/South/proton/data/run000001/corsika_run000001_proton_za020deg_azm000deg_South_4MST_sim_showers_for_trigger_rates.zst

tests/integration_tests/config/simulate_prod_gamma_20_deg.yml

@@ -13,3 +13,6 @@ CTA_SIMPIPE:
     NSHOW: 5
     DATA_DIRECTORY: simtools-output
     OUTPUT_PATH: simtools-output
+  INTEGRATION_TESTS:
+    - OUTPUT_FILE: |
+        test-production/simtel-data/North/gamma/data/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production.simtel.zst

tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml

@@ -14,3 +14,8 @@ CTA_SIMPIPE:
     DATA_DIRECTORY: simtools-output
     OUTPUT_PATH: simtools-output
     PACK_FOR_GRID_REGISTER: true
+  INTEGRATION_TESTS:
+    - OUTPUT_FILE: |
+        directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production.log_hist.tar.gz
+    - OUTPUT_FILE: |
+        directory_for_grid_upload/run000002_gamma_za020deg_azm000deg_North_TestLayout_test-production.simtel.zst

tests/integration_tests/test_applications_from_config.py

@@ -248,7 +248,7 @@ def validate_application_output(config):
             # First check if the output is in the data directory (simtel_array related),
             # Then check if the file is in the output directory (remaining tools).
             logger.info(f"PATH {config['CONFIGURATION']['OUTPUT_PATH']}")
-            logger.info(f"File{integration_test['OUTPUT_FILE']}")
+            logger.info(f"File {integration_test['OUTPUT_FILE']}")
             try:
                 assert (
                     Path(config["CONFIGURATION"]["DATA_DIRECTORY"])