From 76befe142ec6fa8cd51f68497e57c5dfce11c384 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:44:41 -0400 Subject: [PATCH 1/2] Hotfix: Fix generate_workflows.sh optional build flags (#3024) # Description The build flags used when invoking build_all.sh from generate_workflows.sh did were after the verbose flag, which caused them to be ignored if any of the verbose flags (-v, -V, -d) were not set when calling generate_workflows.sh as the `build_all.sh` verbose flag would be set to `--`. This fixes the issue by changing the order of the flags sent to `build_all.sh`. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? Build test on Hercules. # Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes --- workflow/generate_workflows.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index ab40214655..c4f89eef6e 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -2,7 +2,7 @@ ### function _usage() { - cat <<-EOF + cat << EOF This script automates the experiment setup process for the global workflow. Options are also available to update submodules, build the workflow (with specific build flags), specicy which YAMLs and YAML directory to run, and @@ -204,8 +204,7 @@ else echo "The RUNTESTS directory ${_runtests} already exists." echo "Would you like to remove it?" _attempts=0 - while read -r _from_stdin - do + while read -r _from_stdin; do if [[ "${_from_stdin^^}" =~ Y ]]; then rm -rf "${_runtests}" mkdir -p "${_runtests}" @@ -390,7 +389,7 @@ if [[ "${_build}" == "true" ]]; then printf "Building via build_all.sh %s\n\n" "${_build_flags}" # Let the output of build_all.sh go to stdout regardless of verbose options #shellcheck disable=SC2086,SC2248 - ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} + ${HOMEgfs}/sorc/build_all.sh ${_build_flags} ${_verbose_flag} fi # Link the workflow silently unless there's an error From 720eb4c06c7ec325b2175ab5f85177225b81da87 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Tue, 22 Oct 2024 12:33:53 -0400 Subject: [PATCH 2/2] Replace gfs_cyc with an interval (#2928) # Description To facilitate longer and more flexible GFS cadences, the `gfs_cyc` variable is replaced with a specified interval. Up front, this is reflected in a change in the arguments for setup_exp to: ``` --interval ``` Where `n_hours` is the interval (in hours) between gfs forecasts. `n_hours` must be a multiple of 6. If 0, no gfs will be run (only gdas; only valid for cycled mode). The default value is 6 (every cycle). (This is a change from current behavior of 24.) In cycled mode, there is an additional argument to control which cycle will be the first gfs cycle: ``` --sdate_gfs ``` The default if not provided is `--idate` + 6h (first full cycle). This is the same as current behavior when `gfs_cyc` is 6, but may vary from current behavior for other cadences. As part of this change, some of the validation of the dates has been added. `--edate` has also been made optional and defaults to `--idate` if not provided. During `config.base` template-filling, `INTERVAL_GFS` (renamed from `STEP_GFS`) is defined as `--interval` and `SDATE_GFS as `--sdate_gfs`. Some changes were necessary to the gfs verification (metp) job, as `gfs_cyc` was being used downstream by verif-global. That has been removed, and instead workflow will be responsible for only running metp on the correct cycles. This also removes "do nothing" metp tasks that exit immediately, because only the last GFS cycle in a day would actually process verification. Now, metp has its own cycledef and will (a) always runs at 18z, regardless of whether gfs is running at 18z or not, if the interval is less than 24h; (b) use the same cycledef as gfs if the interval is 24h or greater. This is simpler than trying to determine the last gfs cycle of a day when it could change from day to day. To facilitate this change, support for the undocumented rocoto dependency tag `taskvalid` is added, as the metp task needs to know whether the cycle has a gfsarch task or not. metp will trigger on gfsarch completing (as before), or look backwards for the last gfsarch to exist. Additionally, a couple EE2 issues with the metp job are resolved (even though it is not run in ops): - verif-global update replaced `$CDUMP` with `$RUN` - `$DATAROOT` is no longer redefined in the metp job Also corrects some dependency issues with the extractvars job for replay and the replay CI test. Depends on NOAA-EMC/EMC_verif-global#137 Resolves #260 Refs #1299 --------- Co-authored-by: David Huber --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 5 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- ci/cases/weekly/C384_atm3DVar.yaml | 2 +- docs/source/jobs.rst | 2 +- docs/source/setup.rst | 18 ++--- gempak/ush/gfs_meta_comp.sh | 2 +- gempak/ush/gfs_meta_mar_comp.sh | 2 +- jobs/JGFS_ATMOS_VERIFICATION | 6 -- parm/config/gefs/config.base | 8 +-- parm/config/gfs/config.base | 9 +-- parm/config/gfs/config.resources | 4 +- parm/config/gfs/config.wave | 2 +- scripts/exgfs_aero_init_aerosol.py | 16 ++--- sorc/verif-global.fd | 2 +- workflow/applications/applications.py | 17 +---- workflow/applications/gefs.py | 1 - workflow/applications/gfs_cycled.py | 50 +------------ workflow/applications/gfs_forecast_only.py | 2 +- workflow/rocoto/gefs_tasks.py | 2 +- workflow/rocoto/gefs_xml.py | 12 ++-- workflow/rocoto/gfs_cycled_xml.py | 33 +++++++-- workflow/rocoto/gfs_forecast_only_xml.py | 37 +++++++--- workflow/rocoto/gfs_tasks.py | 83 +++++++++++++--------- workflow/rocoto/rocoto.py | 22 ++++++ workflow/rocoto/tasks.py | 3 +- workflow/setup_expt.py | 52 +++++++++++--- 34 files changed, 227 insertions(+), 183 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index d97c9567e9..99ba7c3661 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index 98f0fcfadb..f39031f1a1 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 resensatmos: 48 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index e1b76f0db8..2de5fea7ff 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021032412 edate: 2021032418 nens: 0 - gfs_cyc: 0 + interval: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index 7617e39217..b527903d69 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -14,6 +14,6 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index 7387e55b24..be5ad32238 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021122012 edate: 2021122100 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index b1566d77a0..41a8baa725 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2024022318 edate: 2024022406 nens: 2 - gfs_cyc: 1 + interval: 24 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml index 1475e81ea0..7118dde53f 100644 --- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml +++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 1.0 resensatmos: 96 nens: 2 - gfs_cyc: 1 + interval: 6 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR @@ -17,3 +17,6 @@ arguments: edate: 2020110100 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 + +skip_ci_on_hosts: + - wcoss2 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index e9e6c2b31c..fc09beeacf 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index cdf69f04e0..8ab67a750e 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122118 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index 131ada95d5..6053f73124 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml index 40487f3b47..1a14059ab1 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/ci/cases/weekly/C384_atm3DVar.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 0e3700bf20..2cdecb01de 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -8,7 +8,7 @@ GFS Configuration The sequence of jobs that are run for an end-to-end (analysis+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step. -For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle (See the "run GFS this cycle?" portion of the diagram). The option to run the GFS for all four cycles is available (see the ``gfs_cyc`` variable in configuration file). +For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. An experimental run is different from operations in the following ways: diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 1715899927..e7d5323e40 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -32,7 +32,7 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] + ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] where: @@ -51,12 +51,12 @@ where: * ``$START`` is the start type (warm or cold [default]) * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$PSLOT`` is the name of your experiment [default: test] * ``$CONFIGDIR`` is the path to the ``/config`` folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the system (i.e. 768 for C768) [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the system (i.e. 0.25 for 1/4 degree) [default: 0.; determined based on atmosphere resolution] - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME (but do not use default due to limited space in home directories normally, provide a path to a larger scratch space)] * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] @@ -67,7 +67,7 @@ Atm-only: :: cd workflow - ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir + ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --interval 6 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled: @@ -144,7 +144,8 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] + ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] + [--interval $INTERVAL_GFS] [--sdate_gfs $SDATE_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR] @@ -163,9 +164,10 @@ where: - S2SWA: atm-ocean-ice-wave-aerosols * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$START`` is the start type (warm or cold [default]) - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] + * ``$SDATE_GFS`` cycle to begin GFS forecast [default: $IDATE + 6] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the deterministic forecast [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution] * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192] @@ -184,7 +186,7 @@ Example: :: cd workflow - ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --gfs_cyc 4 + ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --interval 6 Example ``setup_expt.py`` on Orion: diff --git a/gempak/ush/gfs_meta_comp.sh b/gempak/ush/gfs_meta_comp.sh index 36d18d8659..38c15a60c7 100755 --- a/gempak/ush/gfs_meta_comp.sh +++ b/gempak/ush/gfs_meta_comp.sh @@ -24,7 +24,7 @@ device="nc | ${metaname}" export COMIN="gfs.multi" mkdir "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/gempak/ush/gfs_meta_mar_comp.sh b/gempak/ush/gfs_meta_mar_comp.sh index d25fc0dc9a..91f8a48876 100755 --- a/gempak/ush/gfs_meta_mar_comp.sh +++ b/gempak/ush/gfs_meta_mar_comp.sh @@ -15,7 +15,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl export COMIN="gfs.multi" mkdir -p "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/jobs/JGFS_ATMOS_VERIFICATION b/jobs/JGFS_ATMOS_VERIFICATION index 48133364e5..fde0d73b1e 100755 --- a/jobs/JGFS_ATMOS_VERIFICATION +++ b/jobs/JGFS_ATMOS_VERIFICATION @@ -16,12 +16,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "metp" -c "base metp" ## METPCASE : METplus verification use case (g2g1 | g2o1 | pcp1) ############################################################### -# TODO: This should not be permitted as DATAROOT is set at the job-card level. -# TODO: DATAROOT is being used as DATA in metp jobs. This should be rectified in metp. -# TODO: The temporary directory is DATA and is created at the top of the J-Job. -# TODO: remove this line -export DATAROOT=${DATA} - VDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${VRFYBACK_HRS} hours") export VDATE=${VDATE:0:8} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 6cf8488f91..05aabaa323 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -227,7 +227,8 @@ export FHOUT_OCN=3 export FHOUT_ICE=3 # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # set variables needed for use with REPLAY ICs export REPLAY_ICS=@REPLAY_ICS@ @@ -255,11 +256,6 @@ export FHOUT_WAV=3 export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi export ILPOST=1 # gempak output frequency up to F120 export FHMIN_ENKF=${FHMIN_GFS} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 7fa8245057..ccb05abe88 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -283,7 +283,8 @@ export FHOUT_ICE=3 export EUPD_CYC="@EUPD_CYC@" # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # GFS output and frequency export FHMIN_GFS=0 @@ -302,11 +303,7 @@ export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX:-9} export FHMAX_WAV_GFS=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi + # TODO: Change gempak to use standard out variables (#2348) export ILPOST=${FHOUT_HF_GFS} # gempak output frequency up to F120 if (( FHMAX_HF_GFS < 120 )); then diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 26f6126773..79dbb487db 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -988,8 +988,8 @@ case ${step} in threads_per_task=1 walltime_gdas="03:00:00" walltime_gfs="06:00:00" - ntasks=4 - tasks_per_node=4 + ntasks=1 + tasks_per_node=1 export memory="80G" ;; diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index db4eb9f708..ea68508547 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -117,7 +117,7 @@ if [[ "${RUN}" == "gdas" ]]; then export WAVNCYC=4 export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days -elif [[ ${gfs_cyc} -ne 0 ]]; then +elif (( INTERVAL_GFS > 0 )); then export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days else diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index aed6b88647..bc4e495e42 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -11,13 +11,13 @@ --------- This script requires the following environment variables be set beforehand: -CDATE: Initial time in YYYYMMDDHH format -STEP_GFS: Forecast cadence (frequency) in hours -FHMAX_GFS: Forecast length in hours -RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. -ROTDIR: Rotating (COM) directory -USHgfs: Path to global-workflow `ush` directory -PARMgfs: Path to global-workflow `parm` directory +CDATE: Initial time in YYYYMMDDHH format +INTERVAL_GFS: Forecast cadence (frequency) in hours +FHMAX_GFS: Forecast length in hours +RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. +ROTDIR: Rotating (COM) directory +USHgfs: Path to global-workflow `ush` directory +PARMgfs: Path to global-workflow `parm` directory Additionally, the following data files are used: @@ -66,7 +66,7 @@ def main() -> None: # Read in environment variables and make sure they exist cdate = get_env_var("CDATE") - incr = int(get_env_var('STEP_GFS')) + incr = int(get_env_var('INTERVAL_GFS')) fcst_length = int(get_env_var('FHMAX_GFS')) run = get_env_var("RUN") rot_dir = get_env_var("ROTDIR") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index e7e6bc4358..b2ee80cac7 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit e7e6bc43584e0b8911819b8f875cc8ee747db76d +Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a694129e38..ecd320d708 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -68,6 +68,7 @@ def __init__(self, conf: Configuration) -> None: self.nens = base.get('NMEM_ENS', 0) self.fcst_segments = base.get('FCST_SEGMENTS', None) + self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") if not AppConfig.is_monotonic(self.fcst_segments): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') @@ -109,9 +110,6 @@ def _init_finalize(self, conf: Configuration): # Save base in the internal state since it is often needed base = self.configs['_no_run']['base'] - # Get more configuration options into the class attributes - self.gfs_cyc = base.get('gfs_cyc') - # Get task names for the application self.task_names = self.get_task_names() @@ -199,19 +197,6 @@ def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' pass - @staticmethod - def get_gfs_interval(gfs_cyc: int) -> timedelta: - """ - return interval in hours based on gfs_cyc - """ - - gfs_internal_map = {'1': '24H', '2': '12H', '4': '6H'} - - try: - return to_timedelta(gfs_internal_map[str(gfs_cyc)]) - except KeyError: - raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - @staticmethod def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: """ diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c15786a2e8..9d1d5c3dc4 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -42,7 +42,6 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) base_out['RUN'] = 'gefs' return base_out diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 19f4dd607b..da78166ede 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -128,7 +128,7 @@ def _get_app_configs(self): @staticmethod def _update_base(base_in): - return GFSCycledAppConfig.get_gfs_cyc_dates(base_in) + return base_in def get_task_names(self): """ @@ -297,7 +297,7 @@ def get_task_names(self): tasks['enkfgdas'] = enkfgdas_tasks # Add RUN=gfs tasks if running early cycle - if self.gfs_cyc > 0: + if self.interval_gfs > to_timedelta("0H"): tasks['gfs'] = gfs_tasks if self.do_hybvar and 'gfs' in self.eupd_runs: @@ -307,49 +307,3 @@ def get_task_names(self): tasks['enkfgfs'] = enkfgfs_tasks return tasks - - @staticmethod - def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: - """ - Generate GFS dates from experiment dates and gfs_cyc choice - """ - - base_out = base.copy() - - sdate = base['SDATE'] - edate = base['EDATE'] - base_out['INTERVAL'] = to_timedelta(f"{base['assim_freq']}H") - - # Set GFS cycling dates - gfs_cyc = base['gfs_cyc'] - if gfs_cyc != 0: - interval_gfs = AppConfig.get_gfs_interval(gfs_cyc) - hrinc = 0 - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - return base_out diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 93551ac0cc..fb1d2cdb8f 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -78,7 +78,7 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) + base_out['RUN'] = 'gfs' return base_out diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 955f631c8e..e9338c90df 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -472,7 +472,7 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': 'wave_post_grid_mem#member#'} + dep_dict = {'type': 'task', 'name': 'gefs_wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'} diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index b25a73fa6c..a5dfd5140e 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -14,19 +14,19 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] + sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + interval = self._app_config.interval_gfs sdate_str = sdate.strftime("%Y%m%d%H%M") edate_str = edate.strftime("%Y%m%d%H%M") interval_str = timedelta_to_HMS(interval) strings = [] strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - sdate = sdate + interval - if sdate <= edate: - sdate_str = sdate.strftime("%Y%m%d%H%M") - strings.append(f'\t{sdate_str} {edate_str} {interval_str}') + date2 = sdate + interval + if date2 <= edate: + date2_str = date2.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_str} {edate_str} {interval_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index afd663c337..eef77ba7fc 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -24,19 +24,38 @@ def get_cycledefs(self): sdate_str = sdate.strftime("%Y%m%d%H%M") strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - if self._app_config.gfs_cyc != 0: + interval_gfs = self._app_config.interval_gfs + + if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = self._base['EDATE_GFS'] - interval_gfs = self._base['INTERVAL_GFS'] + edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') - sdate_gfs = sdate_gfs + interval_gfs - sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") - if sdate_gfs <= edate_gfs: - strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + date2_gfs = sdate_gfs + interval_gfs + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + if date2_gfs <= edate_gfs: + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') + strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index cf53e685e9..a4d5b0878b 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -12,15 +12,36 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] - edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + sdate_gfs = self._base['SDATE_GFS'] + edate_gfs = self._base['EDATE'] + interval_gfs = self._app_config.interval_gfs strings = [] - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') - - sdate = sdate + interval - if sdate <= edate: - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') + sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") + edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") + interval_gfs_str = timedelta_to_HMS(interval_gfs) + strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + date2 = sdate_gfs + interval_gfs + if date2 <= edate_gfs: + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7c56f25583..82dfb9f1d4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1,6 +1,6 @@ from applications.applications import AppConfig from rocoto.tasks import Tasks -from wxflow import timedelta_to_HMS +from wxflow import timedelta_to_HMS, to_timedelta import rocoto.rocoto as rocoto import numpy as np @@ -39,7 +39,6 @@ def stage_ic(self): def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] - gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], @@ -48,10 +47,10 @@ def prep(self): gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{dump_path}/{self.run}.t@Hz.updated.status.tm00.bufr_d' dep_dict = {'type': 'data', 'data': data} @@ -59,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and self.app_config.interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -89,7 +88,7 @@ def waveinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -189,7 +188,7 @@ def anal(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: @@ -255,7 +254,7 @@ def analcalc(self): dep_dict = {'type': 'task', 'name': f'{self.run}_sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.run in ['gdas']: - dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -330,17 +329,17 @@ def atmanlinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - gfs_cyc = self._base["gfs_cyc"] + interval_gfs = self._base["INTERVAL_GFS"] gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('atmanlinit') @@ -482,7 +481,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -514,7 +513,7 @@ def aeroanlvar(self): deps = [] dep_dict = { 'type': 'task', 'name': f'gdas_aeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { @@ -618,7 +617,7 @@ def esnowrecen(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -644,7 +643,7 @@ def prepoceanobs(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -671,7 +670,7 @@ def marinebmat(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -699,7 +698,7 @@ def marineanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -878,9 +877,9 @@ def _fcst_forecast_only(self): # Calculate offset based on RUN = gfs | gdas interval = None if self.run in ['gfs']: - interval = self._base['INTERVAL_GFS'] + interval = to_timedelta(f"{self._base['INTERVAL_GFS']}H") elif self.run in ['gdas']: - interval = self._base['INTERVAL'] + interval = self._base['assim_freq'] offset = timedelta_to_HMS(-interval) deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_aerosol_init'} @@ -1835,14 +1834,27 @@ def metp(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if self.app_config.interval_gfs < to_timedelta('24H'): + n_lookback = self.app_config.interval_gfs // to_timedelta('6H') + for lookback in range(1, n_lookback + 1): + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + for lookback2 in range(1, lookback): + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) + dep_dict = {'type': 'task', 'name': f'{self.run}_arch', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) metpenvars = self.envars.copy() - if self.app_config.mode in ['cycled']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), - 'EDATE_GFS': self._base.get('EDATE_GFS').strftime("%Y%m%d%H")} - elif self.app_config.mode in ['forecast-only']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), + 'EDATE_GFS': self._base.get('EDATE').strftime("%Y%m%d%H")} metpenvar_dict['METPCASE'] = '#metpcase#' for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1858,7 +1870,7 @@ def metp(self): 'resources': resources, 'dependency': dependencies, 'envars': metpenvars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': 'metp,last_gfs', 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2331,8 +2343,13 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_metp and self.run in ['gfs']: + deps2 = [] + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} - deps.append(rocoto.add_dependency(dep_dict)) + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2358,7 +2375,7 @@ def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2468,7 +2485,7 @@ def atmensanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2495,7 +2512,7 @@ def atmensanlobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2521,7 +2538,7 @@ def atmensanlsol(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2547,7 +2564,7 @@ def atmensanlletkf(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2576,7 +2593,7 @@ def atmensanlfv3inc(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 2a20820da8..7bacf99829 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -183,6 +183,7 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str: 'metatask': _add_task_tag, 'data': _add_data_tag, 'cycleexist': _add_cycle_tag, + 'taskvalid': _add_taskvalid_tag, 'streq': _add_streq_tag, 'strneq': _add_streq_tag, 'sh': _add_sh_tag} @@ -296,6 +297,27 @@ def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: return string +def _add_taskvalid_tag(dep_dict: Dict[str, Any]) -> str: + """ + create a validtask tag + :param dep_dict: dependency key-value parameters + :type dep_dict: dict + :return: Rocoto validtask dependency + :rtype: str + """ + + dep_type = dep_dict.get('type', None) + dep_name = dep_dict.get('name', None) + + if dep_name is None: + msg = f'a {dep_type} name is necessary for {dep_type} dependency' + raise KeyError(msg) + + string = f'<{dep_type} task="{dep_name}"/>' + + return string + + def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: """ create a simple string comparison tag diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a32827377..92ceea73aa 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -57,7 +57,8 @@ def __init__(self, app_config: AppConfig, run: str) -> None: self.nmem = int(self._base['NMEM_ENS_GFS']) else: self.nmem = int(self._base['NMEM_ENS']) - self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 494f5ded4d..f32203e600 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,13 +7,14 @@ import os import glob import shutil -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS +import warnings +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host from wxflow import parse_j2yaml from wxflow import AttrDict -from wxflow import to_datetime, datetime_to_YMDH +from wxflow import to_datetime, to_timedelta, datetime_to_YMDH _here = os.path.dirname(__file__) @@ -115,7 +116,8 @@ def edit_baseconfig(host, inputs, yaml_dict): "@COMROOT@": inputs.comroot, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, - "@gfs_cyc@": inputs.gfs_cyc, + "@INTERVAL_GFS@": inputs.interval, + "@SDATE_GFS@": datetime_to_YMDH(inputs.sdate_gfs), "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } @@ -185,6 +187,19 @@ def input_args(*argv): ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] + def _validate_interval(interval_str): + err_msg = f'must be a non-negative integer multiple of 6 ({interval_str} given)' + try: + interval = int(interval_str) + except ValueError: + raise ArgumentTypeError(err_msg) + + # This assumes the gdas frequency (assim_freq) is 6h + # If this changes, the modulus needs to as well + if interval < 0 or interval % 6 != 0: + raise ArgumentTypeError(err_msg) + return interval + def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') @@ -198,7 +213,8 @@ def _common_args(parser): type=str, required=False, default=os.getenv('HOME')) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) - parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--edate', help='end date experiment', required=False, type=lambda dd: to_datetime(dd)) + parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6', type=_validate_interval, required=False, default=6) parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) @@ -218,8 +234,7 @@ def _gfs_args(parser): def _gfs_cycled_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, - choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser def _gfs_or_gefs_ensemble_args(parser): @@ -232,8 +247,6 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, - choices=[1, 2, 4], default=1, required=False) return parser def _gefs_args(parser): @@ -290,7 +303,28 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) - return parser.parse_args(list(*argv) if len(argv) else None) + inputs = parser.parse_args(list(*argv) if len(argv) else None) + + # Validate dates + if inputs.edate is None: + inputs.edate = inputs.idate + + if inputs.edate < inputs.idate: + raise ArgumentTypeError(f'edate ({inputs.edate}) cannot be before idate ({inputs.idate})') + + # For forecast-only, GFS starts in the first cycle + if not hasattr(inputs, 'sdate_gfs'): + inputs.sdate_gfs = inputs.idate + + # For cycled, GFS starts after the half-cycle + if inputs.sdate_gfs is None: + inputs.sdate_gfs = inputs.idate + to_timedelta("6H") + + if inputs.interval > 0: + if inputs.sdate_gfs < inputs.idate or inputs.sdate_gfs > inputs.edate: + raise ArgumentTypeError(f'sdate_gfs ({inputs.sdate_gfs}) must be between idate ({inputs.idate}) and edate ({inputs.edate})') + + return inputs def query_and_clean(dirname, force_clean=False):