From 86c3eb075f29ac8b7a51038a50384f8185cbc2f2 Mon Sep 17 00:00:00 2001
From: P McCarthy <paddy@ucar.edu>
Date: Fri, 10 Mar 2023 09:39:05 -0500
Subject: [PATCH] [develop] Fixed issue #649 and tested on cheyenne. (#650)

This fixes a problem due to a typo that places the '-n ${NUMTS}' argument before the gefs2lbc_para executable instead of after on Cheyenne for the AQM_LBCS task. The typo causes mpirun to fail on Cheyenne because it's an invalid mpirun argument.

---------

Co-authored-by: Paddy Mccarthy <paddy@cheyenne1.cheyenne.ucar.edu>
Co-authored-by: Paddy Mccarthy <paddy@cheyenne6.cheyenne.ucar.edu>
---
 scripts/exregional_aqm_lbcs.sh | 2 +-
 ush/config_defaults.yaml       | 4 ++++
 ush/machine/cheyenne.yaml      | 2 ++
 ush/machine/hera.yaml          | 1 +
 ush/machine/orion.yaml         | 1 +
 ush/machine/wcoss2.yaml        | 1 +
 6 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh
index 70489cac7e..79dbd1663f 100755
--- a/scripts/exregional_aqm_lbcs.sh
+++ b/scripts/exregional_aqm_lbcs.sh
@@ -215,7 +215,7 @@ Please ensure that you've built this executable."
 #----------------------------------------------------------------------
 #
   PREP_STEP
-  eval ${RUN_CMD_UTILS} -n ${NUMTS} ${exec_fp} ${REDIRECT_OUT_ERR} || \
+  eval ${RUN_CMD_AQMLBC} ${exec_fp} ${REDIRECT_OUT_ERR} || \
     print_err_msg_exit "\
 Call to executable (exec_fp) to generate chemical and GEFS LBCs
 file for RRFS-CMAQ failed:
diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml
index fbb59040a7..fb44af7170 100644
--- a/ush/config_defaults.yaml
+++ b/ush/config_defaults.yaml
@@ -199,6 +199,9 @@ platform:
   # RUN_CMD_AQM:
   # The run command for some AQM tasks.
   #
+  # RUN_CMD_AQMLBC:
+  # The run command for the AQM_LBCS task.
+  #
   #-----------------------------------------------------------------------
   #
   RUN_CMD_SERIAL: ""
@@ -206,6 +209,7 @@ platform:
   RUN_CMD_FCST: ""
   RUN_CMD_POST: ""
   RUN_CMD_AQM: ""
+  RUN_CMD_AQMLBC: ""
 
   #
   #-----------------------------------------------------------------------
diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml
index ed10c964d8..672a8043e8 100644
--- a/ush/machine/cheyenne.yaml
+++ b/ush/machine/cheyenne.yaml
@@ -16,6 +16,8 @@ platform:
   RUN_CMD_POST: mpirun -np $nprocs
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: mpirun -np $nprocs
+  RUN_CMD_AQM: mpirun -np $nprocs
+  RUN_CMD_AQMLBC: mpirun -np ${NUMTS}
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/input_model_data
   TEST_PREGEN_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/FV3LAM_pregen
diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml
index c1b0dfe746..08bc7a49fd 100644
--- a/ush/machine/hera.yaml
+++ b/ush/machine/hera.yaml
@@ -20,6 +20,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL
   RUN_CMD_AQM: srun -n ${nprocs} --export=ALL
+  RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
   SCHED_NATIVE_CMD: --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/input_model_data
diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml
index cd9e898c35..8f03a376e8 100644
--- a/ush/machine/orion.yaml
+++ b/ush/machine/orion.yaml
@@ -20,6 +20,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL
   RUN_CMD_AQM: srun --export=ALL
+  RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
   SCHED_NATIVE_CMD: --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/input_model_data
diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml
index ad4ee2ae95..be3afa4f22 100644
--- a/ush/machine/wcoss2.yaml
+++ b/ush/machine/wcoss2.yaml
@@ -17,6 +17,7 @@ platform:
   RUN_CMD_SERIAL: mpiexec
   RUN_CMD_UTILS: mpiexec -n ${nprocs}
   RUN_CMD_AQM: mpiexec -n ${nprocs} -ppn ${ppn_run_aqm} --cpu-bind core -depth ${omp_num_threads_run_aqm}
+  RUN_CMD_AQMLBC: mpiexec -n ${NUMTS}
   SCHED_NATIVE_CMD: -l place=excl
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/input_model_data