Merge branch 'develop' into new_sfcsub

Fixes ufs-community#933.

CMakeLists.txt

@@ -41,7 +41,6 @@ option(GFS "Enable building GFS-only utilities" OFF)
 # When building the GFS, the following need not be built
 if(GFS)
   message(STATUS "Building utilities specific to the GFS")
-  set(FRENCTOOLS OFF CACHE BOOL "Disable building fre-nctools.fd" FORCE)
   set(GRIDTOOLS OFF CACHE BOOL "Disable building grid_tools.fd" FORCE)
   set(OROG_MASK_TOOLS OFF CACHE BOOL "Disable building orog_mask_tools.fd" FORCE)
   set(SFC_CLIMO_GEN OFF CACHE BOOL "Disable building sfc_climo_gen.fd" FORCE)

README.md

@@ -59,7 +59,7 @@ It also uses the following repositories:
 
 ## Installing
 
-On Orion, Hercules, Jet, Hera and WCOSS2 do the following:
+On Orion, Hercules, Jet, Hera, S4, Gaea and WCOSS2 do the following:
 
 1) Set the 'fixed' directories using the `link_fixdirs.sh`
 script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`,

build_all.sh

@@ -15,10 +15,11 @@ else
   readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 fi
 
+source "${DIR_ROOT}/sorc/machine-setup.sh"
+
 # User Options
 target=${target:-"NULL"}
 compiler=${compiler:-"intel"}
-PW_CSP=${PW_CSP:-} # TODO: This is an implementation from EPIC and consistent with the UFS WM build system.
 
 if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   unset -f module
@@ -27,17 +28,10 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   set -x
 else
   set +x
-  source "${DIR_ROOT}/sorc/machine-setup.sh"
-  if [[ "${target}" == "noaacloud" ]]; then
-    #TODO: This will need to be revisited once the EPIC supported-stacks come online.
-    #TODO: This is a hack due to how the spack-stack module files are generated; there may be a better way to do this.
-    source /contrib/global-workflow/spack-stack/envs/spack_2021.0.3.env
-  else
-    module use "${DIR_ROOT}/modulefiles"
-  fi
+  module use "${DIR_ROOT}/modulefiles"
   module load "build.$target.$compiler" > /dev/null
   module list
- set -x
+  set -x
 fi
 
 # Ensure the submodules have been initialized.

driver_scripts/driver_grid.hercules.sh

@@ -154,8 +154,8 @@ fi
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
-export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
+export TEMP_DIR=/work2/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/work2/noaa/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------
 # Should not need to change anything below here.

fix/link_fixdirs.sh

@@ -9,15 +9,15 @@ set -ex
 #               'nco' (copies data).
 #
 #  $machine - is the machine. Choices are:
-#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4'
+#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4', 'gaea'
 
 RUN_ENVIR=${1}
 machine=${2}
 
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 | gaea )'
     exit 1
 fi
 
@@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     exit 1
 fi
 
-if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules ]; then
+if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaea ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaea )'
     exit 1
 fi
 
@@ -54,6 +54,8 @@ elif [ $machine = "wcoss2" ]; then
     FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix"
 elif [ $machine = "s4" ]; then
     FIX_DIR="/data/prod/glopara/fix"
+elif [ $machine = "gaea" ]; then
+    FIX_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/fix"
 fi
 
 am_ver=${am_ver:-20220805}

modulefiles/build.gaea.intel.lua

@@ -0,0 +1,66 @@
+help([[
+Load environment to compile UFS_UTILS on Gaea using Intel
+]])
+
+prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
+load("hsi")
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.0.6"
+load(pathJoin("nco", nco_ver))
+
+whatis("Description: UFS_UTILS build environment")
+

modulefiles/build.noaacloud.intel.lua

@@ -2,59 +2,21 @@ help([[
 Load environment to compile UFS_UTILS on NOAA CSPs using Intel
 ]])
 
-cmake_ver=os.getenv("cmake_ver") or "3.16.1"
-load(pathJoin("cmake", cmake_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2021.3.0"
-load(pathJoin("intel", hpc_intel_ver))
-
-impi_ver=os.getenv("impi_ver") or "2021.3.0"
-load(pathJoin("impi", impi_ver))
-
-bacio_ver=os.getenv("bacio_ver") or "2.4.1"
-load(pathJoin("bacio", bacio_ver))
-
-g2_ver=os.getenv("g2_ver") or "3.4.5"
-load(pathJoin("g2", g2_ver))
-
-ip_ver=os.getenv("ip_ver") or "4.0.0"
-load(pathJoin("ip", ip_ver))
-
-nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
-load(pathJoin("nemsio", nemsio_ver))
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
 
-sp_ver=os.getenv("sp_ver") or "2.5.0"
-load(pathJoin("sp", sp_ver))
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
+load(pathJoin("stack-intel", stack_intel_ver))
 
-w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
-load(pathJoin("w3emc", w3emc_ver))
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
--- Uncomment when CHGRES_ALL is ON
---sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
---load(pathJoin("sfcio", sfcio_ver))
-
-sigio_ver=os.getenv("sigio_ver") or "2.3.2"
-load(pathJoin("sigio", sigio_ver))
-
-zlib_ver=os.getenv("zlib_ver") or "1.2.11"
-load(pathJoin("zlib", zlib_ver))
-
-png_ver=os.getenv("png_ver") or "1.6.35"
-load(pathJoin("libpng", png_ver))
-
-hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
-load(pathJoin("hdf5", hdf5_ver))
-
-netcdf_ver=os.getenv("netcdf_ver") or "4.6.1"
-load(pathJoin("netcdf", netcdf_ver))
-
-nccmp_ver=os.getenv("nccmp_ver") or "1.8.9.0"
-load(pathJoin("nccmp", nccmp_ver))
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
 
-esmf_ver=os.getenv("esmf_ver") or "8.4.0b08"
-load(pathJoin("esmf", esmf_ver))
+load("common4noaacloud")
 
-nco_ver=os.getenv("nco_ver") or "4.9.1"
-load(pathJoin("nco", nco_ver))
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
 
-whatis("Description: UFS_UTILS build environment")
+whatis("Description: UFS_UTILS build environment on NOAA Cloud")

modulefiles/common4noaacloud.lua

@@ -0,0 +1,60 @@
+help([[ 
+Load environment to compile UFS_UTILS on NOAA CSPs using Intel
+]])
+
+cmake_ver=os.getenv("cmake_ver") or "3.16.1"
+load(pathJoin("cmake", cmake_ver))
+
+hpc_intel_ver=os.getenv("hpc_intel_ver") or "2021.3.0"
+load(pathJoin("intel", hpc_intel_ver))
+
+impi_ver=os.getenv("impi_ver") or "2021.3.0"
+load(pathJoin("impi", impi_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
+load(pathJoin("hdf5", hdf5_ver))
+
+netcdf_ver=os.getenv("netcdf_ver") or "4.6.1"
+load(pathJoin("netcdf", netcdf_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "4.9.1"
+load(pathJoin("nco", nco_ver))
+
+whatis("Description: UFS_UTILS build environment")

reg_tests/global_cycle/C192.lndincsoilnoahmp.sh → reg_tests/global_cycle/C192.gsi_lndincsoilnoahmp.sh

@@ -29,8 +29,6 @@ export OCNRES=99999
 
 export COMIN=$HOMEreg/input_data_noahmp
 
-export LND_SOI_FILE=$COMIN/sfcincr_gsi
-
 export JCAP=1534
 export LONB=3072
 export LATB=1536
@@ -40,6 +38,7 @@ export use_ufo=.true.
 
 export DO_SFCCYCLE=".FALSE." 
 export DO_LNDINC=".TRUE." 
+export DO_SOI_INC_GSI=".true."
 
 export VERBOSE=YES
 export CYCLVARS=FSNOL=-2.,FSNOS=99999.,
@@ -49,7 +48,7 @@ $HOMEgfs/ush/global_cycle_driver.sh
 iret=$?
 if [ $iret -ne 0 ]; then
   set +x
-  echo "<<< C192 LANDINC SOIL NOAHMP CYCLE TEST FAILED. >>>"
+  echo "<<< C192 GSI based LANDINC SOIL NOAHMP CYCLE TEST FAILED. >>>"
   exit $iret
 fi
 
@@ -60,7 +59,19 @@ for files in *tile*.nc
 do
   if [ -f $files ]; then
     echo CHECK $files
-    $NCCMP -dmfqS $files $HOMEreg/baseline_data/c192.lndincsoilnoahmp/$files
+    $NCCMP -dmfqS $files $HOMEreg/baseline_data/c192.gsi_lndincsoilnoahmp/$files
+    iret=$?
+    if [ $iret -ne 0 ]; then
+      test_failed=1
+    fi
+  fi
+done
+
+for files in *gaussian_interp*
+do
+  if [ -f $files ]; then
+    echo CHECK $files
+    $NCCMP -dmfqS $files $HOMEreg/baseline_data/c192.gsi_lndincsoilnoahmp/$files
     iret=$?
     if [ $iret -ne 0 ]; then
       test_failed=1
@@ -72,15 +83,15 @@ set +x
 if [ $test_failed -ne 0 ]; then
   echo
   echo "**********************************************"
-  echo "<<< C192 LANDINC SOIL-NOAHMP CYCLE TEST FAILED. >>>"
+  echo "<<< C192 GSI based LANDINC SOIL-NOAHMP CYCLE TEST FAILED. >>>"
   echo "**********************************************"
   if [ "$UPDATE_BASELINE" = "TRUE" ]; then
     $HOMEgfs/reg_tests/update_baseline.sh $HOMEreg "c192.lndincsoilnoahmp" $commit_num
   fi
 else
   echo
   echo "*****************************************"
-  echo "<<< C192 LANDINC SOIL-NOAHMP CYCLE TEST PASSED. >>>"
+  echo "<<< C192 GSI based LANDINC SOIL-NOAHMP CYCLE TEST PASSED. >>>"
   echo "*****************************************"
 fi