GAEA support, authoritative branches, non-nested global grids

.gitmodules

@@ -1,22 +1,15 @@
 [submodule "ufs-weather-model"]
     path = sorc/hafs_forecast.fd
-    url = https://github.com/NOAA-GSL/ufs-weather-model.git
-    branch = WaterInTheWest
-#    url = https://github.com/hafs-community/ufs-weather-model.git
-#    branch = support/hafs.v1.1.0
-#   url = https://github.com/ufs-community/ufs-weather-model.git
-#   branch = develop
+    url = https://github.com/ufs-community/ufs-weather-model
+    branch = develop
 [submodule "UFS_UTILS"]
     path = sorc/hafs_utils.fd
-#   url = https://github.com/hafs-community/UFS_UTILS.git
-#   branch = support/HAFS
-    url = https://github.com/ufs-community/UFS_UTILS.git
+    url = https://github.com/ufs-community/UFS_UTILS
     branch = develop
 [submodule "gfdl-tracker"]
     path = sorc/hafs_tracker.fd
-    url = https://github.com/hafs-community/gfdl-tracker.git
-#   branch = support/HAFS
-    branch = support/hafs.v1.1.0
+    url = https://github.com/hafs-community/gfdl-tracker
+    branch = support/HAFS
 [submodule "sorc/hafs_gsi.fd"]
     path = sorc/hafs_gsi.fd
     url = https://github.com/hafs-community/GSI.git

jobs/JHAFS_PRODUCT

@@ -70,7 +70,7 @@ fi
 mkdir -p $DATA
 cd $DATA
 
-if [ ${gtype} = nest ]; then
+if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
   ngrids=$((${nest_grids} + 1))
 else
   ngrids=${nest_grids}

modulefiles/hafs.gaea_c5.lua

@@ -0,0 +1,79 @@
+help([[
+  This module loads libraries required for building and running HAFS
+  on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis("Description: HAFS Application environment")
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+-- load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+prepend_path("MODULEPATH", "/lustre/f2/dev/Samuel.Trahan/hafs/modulefiles/")
+
+miniconda_ver=os.getenv("miniconda_ver") or "23.5.2-hafs"
+load(pathJoin("miniconda", miniconda_ver))
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+-- stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+-- load(pathJoin("stack-python", stack_python_ver))
+
+local ufs_modules = {
+  {["jasper"]          = "2.0.32"},
+  {["zlib"]            = "1.2.13"},
+  {["libpng"]          = "1.6.37"},
+  {["hdf5"]            = "1.14.0"},
+  {["netcdf-c"]        = "4.9.2"},
+  {["netcdf-fortran"]  = "4.6.0"},
+  {["parallelio"]      = "2.5.10"},
+  {["esmf"]            = "8.4.2"},
+  {["fms"]             = "2023.01"},
+  {["bacio"]           = "2.4.1"},
+  {["crtm"]            = "2.4.0"},
+  {["g2"]              = "3.4.5"},
+  {["g2tmpl"]          = "1.10.2"},
+  {["ip"]              = "3.3.3"},
+  {["nemsio"]          = "2.5.4"},
+  {["sp"]              = "2.3.3"},
+  {["w3emc"]           = "2.9.2"},
+  {["w3nco"]           = "2.4.1"},
+  {["gftl-shared"]     = "1.5.0"},
+-- no yafyaml
+  {["mapl"]            = "2.35.2-esmf-8.4.2"},
+  {["bufr"]            = "11.7.0"},
+  {["sfcio"]           = "1.4.1"},
+  {["sigio"]           = "2.3.2"},
+-- no szip
+  {["wrf-io"]          = "1.2.0"},
+  {["prod-util"]       = "1.2.2"},
+  {["grib-util"]       = "1.2.3"},
+  {["wgrib2"]          = "2.0.8"},
+-- no gempak
+  {["nco"]             = "5.0.6"},
+  {["cdo"]             = "2.0.5"},
+}
+
+for i = 1, #ufs_modules do
+  for name, default_version in pairs(ufs_modules[i]) do
+    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+    load(pathJoin(name, os.getenv(env_version_name) or default_version))
+  end
+end
+
+load("rocoto")
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CMAKE_C_COMPILER", "cc")
+setenv("CMAKE_CXX_COMPILER", "CC")
+setenv("CMAKE_Fortran_COMPILER", "ftn")
+setenv("CMAKE_Platform", "gaea_c5.intel")

modulefiles/hafs.hera.lua

@@ -6,122 +6,65 @@ prepend_path("MODULEPATH", "/contrib/sutils/modulefiles")
 load("sutils")
 load("hpss")
 
-cmake_ver=os.getenv("cmake_ver") or "3.20.1"
-load(pathJoin("cmake", cmake_ver))
-
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles")
-load(pathJoin("python","wcoss2_env"))
-
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
-hpc_ver=os.getenv("hpc_ver") or "1.2.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2"
-load(pathJoin("hpc-impi", hpc_impi_ver))
-
-jasper_ver=os.getenv("jasper_ver") or "2.0.25"
-load(pathJoin("jasper", jasper_ver))
-
-zlib_ver=os.getenv("zlib_ver") or "1.2.11"
-load(pathJoin("zlib", zlib_ver))
-
-zstd_ver=os.getenv("zstd_ver") or "1.5.0"
-load(pathJoin("zstd", zstd_ver))
-
-libpng_ver=os.getenv("libpng_ver") or "1.6.37"
-load(pathJoin("libpng", libpng_ver))
-
-hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
-load(pathJoin("hdf5", hdf5_ver))
-
-netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
-load(pathJoin("netcdf", netcdf_ver))
-
-pio_ver=os.getenv("pio_ver") or "2.5.7"
-load(pathJoin("pio", pio_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.3.0b09"
-load(pathJoin("esmf", esmf_ver))
-
-fms_ver=os.getenv("fms_ver") or "2022.04"
-load(pathJoin("fms",fms_ver))
-
-bacio_ver=os.getenv("bacio_ver") or "2.4.1"
-load(pathJoin("bacio", bacio_ver))
-
-crtm_ver=os.getenv("crtm_ver") or "2.4.0"
-load(pathJoin("crtm", crtm_ver))
-
-g2_ver=os.getenv("g2_ver") or "3.4.5"
-load(pathJoin("g2", g2_ver))
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
 
-g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2"
-load(pathJoin("g2tmpl", g2tmpl_ver))
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
 
-ip_ver=os.getenv("ip_ver") or "3.3.3"
-load(pathJoin("ip", ip_ver))
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-nemsio_ver_ver=os.getenv("nemsio_ver") or "2.5.4"
-load(pathJoin("nemsio", nemsio_ver))
-
-sp_ver=os.getenv("sp_ver") or "2.3.3"
-load(pathJoin("sp", sp_ver))
-
-w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
-load(pathJoin("w3emc", w3emc_ver))
-
-w3nco_ver=os.getenv("w3nco_ver") or "2.4.1"
-load(pathJoin("w3nco", w3nco_ver))
-
-gftl_shared_ver=os.getenv("gftl_shared_ver") or "v1.5.0"
-load(pathJoin("gftl-shared", gftl_shared_ver))
-
-yafyaml_ver=os.getenv("yafyaml_ver") or "v0.5.1"
-load(pathJoin("yafyaml", yafyaml_ver))
-
-mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09"
-load(pathJoin("mapl", mapl_ver))
-
-bufr_ver=os.getenv("bufr_ver") or "11.4.0"
-load(pathJoin("bufr", bufr_ver))
-
-sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
-load(pathJoin("sfcio", sfcio_ver))
-
-sigio_ver=os.getenv("sigio_ver") or "2.3.2"
-load(pathJoin("sigio", sigio_ver))
-
-szip_ver=os.getenv("szip_ver") or "2.1.1"
-load(pathJoin("szip", szip_ver))
-
-wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
-load(pathJoin("wrf_io", wrf_io_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.4"
-load(pathJoin("grib_util", grib_util_ver))
-
-wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8"
-load(pathJoin("wgrib2", wgrib2_ver))
-setenv("WGRIB2", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/impi-2018.0.4/wgrib2/2.0.8/bin/wgrib2")
-
-gempak_ver=os.getenv("gempak_ver") or "7.4.2"
-load(pathJoin("gempak", gempak_ver))
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
 
--- nco_ver=os.getenv("nco_ver") or "5.0.6"
-nco_ver=os.getenv("nco_ver") or "4.9.3"
-load(pathJoin("nco", nco_ver))
+local hafs_modules = {
+  {["jasper"]          = "2.0.32"},
+  {["zlib"]            = "1.2.13"},
+  {["zstd"]            = "1.5.2"},
+  {["libpng"]          = "1.6.37"},
+  {["hdf5"]            = "1.14.0"},
+  {["netcdf-c"]        = "4.9.2"},
+  {["netcdf-fortran"]  = "4.6.0"},
+  {["parallelio"]      = "2.5.10"},
+  {["esmf"]            = "8.4.2"},
+  {["fms"]             = "2023.01"},
+  {["bacio"]           = "2.4.1"},
+  {["crtm"]            = "2.4.0"},
+  {["g2"]              = "3.4.5"},
+  {["g2tmpl"]          = "1.10.2"},
+  {["ip"]              = "4.3.0"},
+  {["nemsio"]          = "2.5.4"},
+  {["sp"]              = "2.3.3"},
+  {["w3emc"]           = "2.10.0"},
+  {["w3nco"]           = "2.4.1"},
+  {["gftl-shared"]     = "1.5.0"},
+  {["yafyaml"]         = "0.5.1"},
+  {["mapl"]            = "2.35.2-esmf-8.4.2"},
+  {["bufr"]            = "12.0.0"},
+  {["sfcio"]           = "1.4.1"},
+  {["sigio"]           = "2.3.2"},
+  {["szip"]            = "2.1"},
+  {["wrf-io"]          = "1.2.0"},
+  {["prod_util"]       = "1.2.2"},
+  {["grib-util"]       = "1.3.0"},
+  {["wgrib2"]          = "2.0.8"},
+  {["gempak"]          = "7.4.2"},
+  {["nco"]             = "5.0.6"},
+  {["cdo"]             = "2.0.5"},
+  {["rocoto"]          = "1.3.3"},
+  {["scotch"]          = "7.0.4"},
+}
+
+for i = 1, #hafs_modules do
+  for name, default_version in pairs(hafs_modules[i]) do
+    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+    load(pathJoin(name, os.getenv(env_version_name) or default_version))
+  end
+end
 
-cdo_ver=os.getenv("cdo_ver") or "1.9.8"
-load(pathJoin("cdo", cdo_ver))
 
-rocoto_ver=os.getenv("rocoto_ver") or "1.3.3"
-load(pathJoin("rocoto", rocoto_ver))
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles")
+load(pathJoin("python","wcoss2_env"))
 
 setenv("CMAKE_C_COMPILER", "mpiicc")
 setenv("CMAKE_CXX_COMPILER", "mpiicpc")

parm/WaterInTheWest.conf

@@ -69,9 +69,9 @@ jend_nest=1294
 regional_esg=no
 
 [forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
 
 # For the global domain if it is a global or global-nesting experiment
 glob_k_split=1
@@ -111,7 +111,8 @@ npz=64
 is_moving_nest=.false.
 
 quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
+write_tasks_per_group=360
 output_history=.true.
 
 output_grid=regional_latlon,regional_latlon
@@ -160,13 +161,14 @@ lightning_threat=.true.
 satpost=.false.
 post_gridspecs=auto,auto
 trak_gridspecs={post_gridspecs}
-postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
-postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
+# postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
+# postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
 
 [atm_post_ens]
 satpost_ens=.false.
 post_gridspecs_ens=auto,auto
 trak_gridspecs_ens={post_gridspecs}
 
 [rocotostr]
-FORECAST_RESOURCES=FORECAST_RESOURCES_2460PE
+FORECAST_RESOURCES=FORECAST_RESOURCES_3060PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES

parm/WaterInTheWestC192.conf

@@ -69,9 +69,9 @@ jend_nest=322
 regional_esg=no
 
 [forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
 
 # For the global domain if it is a global or global-nesting experiment
 glob_k_split=1
@@ -113,7 +113,7 @@ npz=64
 is_moving_nest=.false.
 
 quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
 output_history=.true.
 
 output_grid=regional_latlon,regional_latlon
@@ -174,3 +174,4 @@ trak_gridspecs_ens={post_gridspecs}
 
 [rocotostr]
 FORECAST_RESOURCES=FORECAST_RESOURCES_600PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES

parm/WaterInTheWestC384.conf

@@ -69,9 +69,9 @@ jend_nest=648
 regional_esg=no
 
 [forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
 
 # For the global domain if it is a global or global-nesting experiment
 glob_k_split=1
@@ -111,7 +111,8 @@ npz=64
 is_moving_nest=.false.
 
 quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
+write_tasks_per_group=120
 output_history=.true.
 
 output_grid=regional_latlon,regional_latlon
@@ -171,4 +172,5 @@ post_gridspecs_ens=auto,auto
 trak_gridspecs_ens={post_gridspecs}
 
 [rocotostr]
-FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE
+FORECAST_RESOURCES=FORECAST_RESOURCES_1440PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES