update cmeps with fix for mediator history files; use esmf-managed th…

…reading for cpld_bmark control and restart (was #1483); Software module updates in hpc-stack for Hera (intel, gnu) (was #1468) (#1486) * update CMEPS submodule * bmark cpld tests use esmf-managed threading by default * remove version w/o esmf-managed threading * update hera hpc stack locations: intel/gnu Co-authored-by: Brian Curtis <brian.curtis@noaa.gov> Co-authored-by: jkbk2004 <jong.kim@noaa.gov> Co-authored-by: zach1221 <99902696+zach1221@users.noreply.github.com>
ufs-community · Nov 16, 2022 · cd35139 · cd35139
1 parent 8d90fe7
commit cd35139
Show file tree

Hide file tree

Showing 18 changed files with 4,173 additions and 4,454 deletions.
diff --git a/CMEPS-interface/CMEPS b/CMEPS-interface/CMEPS
diff --git a/modulefiles/ufs_hera.gnu.lua b/modulefiles/ufs_hera.gnu.lua
@@ -12,12 +12,15 @@ load("sutils")
 cmake_ver=os.getenv("cmake_ver") or "3.20.1"
 load(pathJoin("cmake", cmake_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+gnu_ver=os.getenv("gnu_ver") or "9.2.0"
+load(pathJoin("gnu", gnu_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack")
+
+hpc_ver=os.getenv("hpc_ver") or "1.2.0"
 load(pathJoin("hpc", hpc_ver))
 
-hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2.0"
+hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2"
 load(pathJoin("hpc-gnu", hpc_gnu_ver))
 
 hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2"

diff --git a/modulefiles/ufs_hera.gnu_debug.lua b/modulefiles/ufs_hera.gnu_debug.lua
@@ -12,12 +12,15 @@ load("sutils")
 cmake_ver=os.getenv("cmake_ver") or "3.20.1"
 load(pathJoin("cmake", cmake_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+gnu_ver=os.getenv("gnu_ver") or "9.2.0"
+load(pathJoin("gnu", gnu_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack")
+
+hpc_ver=os.getenv("hpc_ver") or "1.2.0"
 load(pathJoin("hpc", hpc_ver))
 
-hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2.0"
+hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2"
 load(pathJoin("hpc-gnu", hpc_gnu_ver))
 
 hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2"

diff --git a/modulefiles/ufs_hera.intel.lua b/modulefiles/ufs_hera.intel.lua
@@ -12,9 +12,15 @@ load("sutils")
 cmake_ver=os.getenv("cmake_ver") or "3.20.1"
 load(pathJoin("cmake", cmake_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+intel_ver=os.getenv("intel_ver") or "2022.1.2"
+load(pathJoin("intel", intel_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
+impi_ver=os.getenv("impi_ver") or "2022.1.2"
+load(pathJoin("impi", impi_ver))
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
+
+hpc_ver=os.getenv("hpc_ver") or "1.2.0"
 load(pathJoin("hpc", hpc_ver))
 
 hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"

diff --git a/modulefiles/ufs_hera.intel_debug.lua b/modulefiles/ufs_hera.intel_debug.lua
@@ -12,9 +12,15 @@ load("sutils")
 cmake_ver=os.getenv("cmake_ver") or "3.20.1"
 load(pathJoin("cmake", cmake_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+intel_ver=os.getenv("intel_ver") or "2022.1.2"
+load(pathJoin("intel", intel_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
+impi_ver=os.getenv("impi_ver") or "2022.1.2"
+load(pathJoin("impi", impi_ver))
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
+
+hpc_ver=os.getenv("hpc_ver") or "1.2.0"
 load(pathJoin("hpc", hpc_ver))
 
 hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"

diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss2.intel.log b/tests/RegressionTests_wcoss2.intel.log
diff --git a/tests/ci/Jenkinsfile b/tests/ci/Jenkinsfile
@@ -72,7 +72,7 @@ pipeline {
 
                     stage('Stage Test Data') {
                         steps {
-                            sh 'docker volume rm --force DataVolume && docker run --detach --rm --volume DataVolume:/tmp noaaepic/input-data:20220414'
+                            sh 'docker volume rm --force DataVolume && docker run --detach --rm --volume DataVolume:/tmp noaaepic/input-data:20221101'
                         }
                     }
 

diff --git a/tests/rt.conf b/tests/rt.conf
@@ -17,7 +17,6 @@ RUN     | cpld_restart_c192_p8
 
 RUN     | cpld_bmark_p8                                                                                                           | - wcoss2.intel jet.intel cheyenne.intel acorn.intel | fv3 |
 RUN     | cpld_restart_bmark_p8                                                                                                   | - wcoss2.intel jet.intel cheyenne.intel acorn.intel |     | cpld_bmark_p8
-RUN     | cpld_bmark_esmfthreads_p8                                                                                               | - wcoss2.intel jet.intel cheyenne.intel acorn.intel | fv3 |
 
 COMPILE | -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8                                                                         |                                         | fv3 |
 RUN     | cpld_control_noaero_p8                                                                                                  |                                         | fv3 |

diff --git a/tests/tests/cpld_bmark_esmfthreads_p8 b/tests/tests/cpld_bmark_esmfthreads_p8
diff --git a/tests/tests/cpld_bmark_p8 b/tests/tests/cpld_bmark_p8
@@ -54,6 +54,8 @@ export LIST_FILES="sfcf006.nc \
 export_fv3
 export_cpl
 
+export NEMS_CONFIGURE=nems.configure.cpld_esmfthreads.IN
+
 export SYEAR=2013
 export SMONTH=04
 export SDAY=01
@@ -75,14 +77,22 @@ export RUN_BEG="${SYEAR}${SMONTH}${SDAY} $(printf "%02d" $(( ${SHOUR}  )))0000"
 export TASKS=$TASKS_cpl_bmrk
 export INPES=$INPES_cpl_bmrk
 export JNPES=$JNPES_cpl_bmrk
-export THRD=$THRD_cpl_bmrk
-export WRTTASK_PER_GROUP=$WPG_cpl_bmrk
+export THRD=1
+export ATM_compute_tasks=$(( INPES * JNPES * NTILES * THRD_cpl_bmrk ))
+export WRTTASK_PER_GROUP=$(( WPG_cpl_bmrk * THRD_cpl_bmrk ))
 export WRITE_DOPOST=.true.
 export WLCLK=${WLCLK_cpl_bmrk:-$WLCLK}
 
-OCN_tasks=$OCN_tasks_cpl_bmrk
-ICE_tasks=$ICE_tasks_cpl_bmrk
-WAV_tasks=$WAV_tasks_cpl_bmrk
+export OCN_tasks=$(( OCN_tasks_cpl_bmrk * OCN_thrds_cpl_bmrk ))
+export ICE_tasks=$(( ICE_tasks_cpl_bmrk * ICE_thrds_cpl_bmrk ))
+export WAV_tasks=$(( WAV_tasks_cpl_bmrk * WAV_thrds_cpl_bmrk ))
+
+export atm_omp_num_threads=$THRD_cpl_bmrk
+export chm_omp_num_threads=$atm_omp_num_threads
+export med_omp_num_threads=$atm_omp_num_threads
+export ocn_omp_num_threads=$OCN_thrds_cpl_bmrk
+export ice_omp_num_threads=$ICE_thrds_cpl_bmrk
+export wav_omp_num_threads=$WAV_thrds_cpl_bmrk
 
 # atm/ocn/ice resolution
 export ATMRES=C384
@@ -146,7 +156,3 @@ export RST_BEG=$RUN_BEG
 export RST_2_BEG=$RUN_BEG
 
 export FV3_RUN=cpld_control_run.IN
-
-if [[ $MACHINE_ID = cheyenne.* ]]; then
-  TPN=15
-fi
diff --git a/tests/tests/cpld_restart_bmark_p8 b/tests/tests/cpld_restart_bmark_p8
@@ -54,6 +54,8 @@ export LIST_FILES="sfcf006.nc \
 export_fv3
 export_cpl
 
+export NEMS_CONFIGURE=nems.configure.cpld_esmfthreads.IN
+
 export SYEAR=2013
 export SMONTH=04
 export SDAY=01
@@ -78,13 +80,22 @@ export RUN_BEG="${SYEAR}${SMONTH}${SDAY} $(printf "%02d" $(( ${FHROT}+${SHOUR}
 export TASKS=$TASKS_cpl_bmrk
 export INPES=$INPES_cpl_bmrk
 export JNPES=$JNPES_cpl_bmrk
-export THRD=$THRD_cpl_bmrk
+export THRD=1
+export ATM_compute_tasks=$(( INPES * JNPES * NTILES * THRD_cpl_bmrk ))
+export WRTTASK_PER_GROUP=$(( WPG_cpl_bmrk * THRD_cpl_bmrk ))
 export WRITE_DOPOST=.true.
-export WRTTASK_PER_GROUP=$WPG_cpl_bmrk
+export WLCLK=${WLCLK_cpl_bmrk:-$WLCLK}
+
+export OCN_tasks=$(( OCN_tasks_cpl_bmrk * OCN_thrds_cpl_bmrk ))
+export ICE_tasks=$(( ICE_tasks_cpl_bmrk * ICE_thrds_cpl_bmrk ))
+export WAV_tasks=$(( WAV_tasks_cpl_bmrk * WAV_thrds_cpl_bmrk ))
 
-OCN_tasks=$OCN_tasks_cpl_bmrk
-ICE_tasks=$ICE_tasks_cpl_bmrk
-WAV_tasks=$WAV_tasks_cpl_bmrk
+export atm_omp_num_threads=$THRD_cpl_bmrk
+export chm_omp_num_threads=$atm_omp_num_threads
+export med_omp_num_threads=$atm_omp_num_threads
+export ocn_omp_num_threads=$OCN_thrds_cpl_bmrk
+export ice_omp_num_threads=$ICE_thrds_cpl_bmrk
+export wav_omp_num_threads=$WAV_thrds_cpl_bmrk
 
 # atm/ocn/ice resolution
 export ATMRES=C384
@@ -162,7 +173,3 @@ export RST_BEG=$RUN_BEG
 export RST_2_BEG=$RUN_BEG
 
 export FV3_RUN=cpld_control_run.IN
-
-if [[ $MACHINE_ID = cheyenne.* ]]; then
-  TPN=15
-fi