Merge remote-tracking branch 'origin/develop' into feature/ss_180

.gitignore

@@ -79,6 +79,7 @@ parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
+parm/ufs/post_itag_gfs
 parm/wafs
 
 # Ignore sorc and logs folders from externals

docs/source/errors_faq.rst

@@ -2,6 +2,14 @@
 Common Errors Known Issues
 ==========================
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Error: Reserved Variables Causing Workflow Issues
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Several variables are reserved in the workflow and should not be used as environment variables in your shell. Some of the common ones include (but are not limited to):
+``HOMEgfs``, ``machine``, ``ROTDIR``, ``COMROT``, ``COMROOT``, ``COMOUT``, ``COMIN``, ``STMP``, ``PTMP``, ``DATAROOT``, ``DATA``, ``ACCOUNT``, ``PDY``, ``cyc``, ``RUN``, etc.
+If you are using any of these variables in your shell, you may encounter unexpected behavior in the workflow.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Error: "ImportError" message when running setup script
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -42,4 +50,3 @@ Issue: Directory name change for EnKF folder in ROTDIR
 **Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
 
 **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle.
-

docs/source/index.rst

@@ -1,6 +1,6 @@
 
 ###############
-Global Workflow 
+Global Workflow
 ###############
 
 **Global-workflow** is the end-to-end workflow designed to run global configurations of medium range weather forecasting for the UFS weather model. It supports both development and operational implementations. In its current format it supports the Global Forecast System (GFS) and the Global Ensemble Forecast System (GEFS) configurations
@@ -31,7 +31,7 @@ GitHub updates: Users should adjust their "Watch" settings for this repo so they
 Table of Contents
 =================
 
-.. toctree:: 
+.. toctree::
    :numbered:
    :maxdepth: 3
 
@@ -42,3 +42,4 @@ Table of Contents
    output.rst
    run.rst
    noaa_csp.rst
+   errors_faq.rst

docs/source/run.rst

@@ -13,4 +13,3 @@ Here we will show how you can run an experiment using the Global Workflow. The G
    start.rst
    monitor_rocoto.rst
    view.rst
-   errors_faq.rst

env/AWSPW.env

@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    export APRUN="${launcher} -n ${ntasks}"
+    APRUN_default="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing AWSPW.env"
     exit 2
@@ -53,7 +53,7 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN}"
+    export APRUN_NP="${APRUN_default}"
 
     export NTHREADS_DWN=${threads_per_task_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
@@ -71,33 +71,33 @@ elif [[ "${step}" = "oceanice_products" ]]; then
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN}"
+    export APRUN_ECEN="${APRUN_default}"
 
     export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN}"
+    export APRUN_CALCINC="${APRUN_default}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
     export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN}"
+    export APRUN_ESFC="${APRUN_default}"
 
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN}"
+    export APRUN_CYCLE="${APRUN_default}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN}"
+    export APRUN_EPOS="${APRUN_default}"
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN}"
+    export MPIRUN="${APRUN_default}"
 
 fi

env/AZUREPW.env

@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    APRUN="${launcher} -n ${ntasks}"
+    APRUN_default="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing AZUREPW.env"
     exit 2
@@ -43,13 +43,61 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN}"
+    export APRUN_NP="${APRUN_default}"
 
     export NTHREADS_DWN=${threads_per_task_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
 fi

env/GAEA.env

@@ -28,7 +28,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
     # This may be useful when Gaea is fully ported, so ignore SC warning
     # shellcheck disable=SC2034
-    APRUN="${launcher} -n ${ntasks}"
+    APRUN_default="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing GAEA.env"
     exit 2
@@ -51,7 +51,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_GSI=${NTHREADSmax}
-    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+    export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
@@ -69,7 +69,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 elif [[ "${step}" = "sfcanl" ]]; then
 
     export NTHREADS_CYCLE=${threads_per_task:-14}
-    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
+    export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
 elif [[ "${step}" = "fcst" ]]; then
 
@@ -82,7 +82,7 @@ elif [[ "${step}" = "fcst" ]]; then
 elif [[ "${step}" = "upp" ]]; then
 
     export NTHREADS_UPP=${NTHREADS1}
-    export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"
+    export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
 
 elif [[ "${step}" = "atmos_products" ]]; then
 
@@ -96,6 +96,6 @@ elif [[ "${step}" = "oceanice_products" ]]; then
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}"
+    export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
 
 fi

env/GOOGLEPW.env

@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    APRUN="${launcher} -n ${ntasks}"
+    APRUN_default="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env"
     exit 2
@@ -57,7 +57,7 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN}"
+    export APRUN_NP="${APRUN_default}"
 
     export NTHREADS_DWN=${threads_per_task_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
@@ -75,29 +75,29 @@ elif [[ "${step}" = "oceanice_products" ]]; then
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN}"
+    export APRUN_ECEN="${APRUN_default}"
 
     export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN}"
+    export APRUN_CALCINC="${APRUN_default}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
     export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN}"
+    export APRUN_ESFC="${APRUN_default}"
 
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN}"
+    export APRUN_CYCLE="${APRUN_default}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN}"
+    export APRUN_EPOS="${APRUN_default}"
 
 elif [[ "${step}" = "fit2obs" ]]; then