diff --git a/.gitmodules b/.gitmodules
index fc5360a86..03b471991 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,22 +1,15 @@
[submodule "ufs-weather-model"]
path = sorc/hafs_forecast.fd
- url = https://github.com/NOAA-GSL/ufs-weather-model.git
- branch = WaterInTheWest
-# url = https://github.com/hafs-community/ufs-weather-model.git
-# branch = support/hafs.v1.1.0
-# url = https://github.com/ufs-community/ufs-weather-model.git
-# branch = develop
+ url = https://github.com/ufs-community/ufs-weather-model
+ branch = develop
[submodule "UFS_UTILS"]
path = sorc/hafs_utils.fd
-# url = https://github.com/hafs-community/UFS_UTILS.git
-# branch = support/HAFS
- url = https://github.com/ufs-community/UFS_UTILS.git
+ url = https://github.com/ufs-community/UFS_UTILS
branch = develop
[submodule "gfdl-tracker"]
path = sorc/hafs_tracker.fd
- url = https://github.com/hafs-community/gfdl-tracker.git
-# branch = support/HAFS
- branch = support/hafs.v1.1.0
+ url = https://github.com/hafs-community/gfdl-tracker
+ branch = support/HAFS
[submodule "sorc/hafs_gsi.fd"]
path = sorc/hafs_gsi.fd
url = https://github.com/hafs-community/GSI.git
diff --git a/jobs/JHAFS_PRODUCT b/jobs/JHAFS_PRODUCT
index 8de58a716..2eaffd7d8 100755
--- a/jobs/JHAFS_PRODUCT
+++ b/jobs/JHAFS_PRODUCT
@@ -70,7 +70,7 @@ fi
mkdir -p $DATA
cd $DATA
-if [ ${gtype} = nest ]; then
+if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
ngrids=$((${nest_grids} + 1))
else
ngrids=${nest_grids}
diff --git a/modulefiles/hafs.gaea_c5.lua b/modulefiles/hafs.gaea_c5.lua
new file mode 100644
index 000000000..b7cebcf58
--- /dev/null
+++ b/modulefiles/hafs.gaea_c5.lua
@@ -0,0 +1,79 @@
+help([[
+ This module loads libraries required for building and running HAFS
+ on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis("Description: HAFS Application environment")
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+-- load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+prepend_path("MODULEPATH", "/lustre/f2/dev/Samuel.Trahan/hafs/modulefiles/")
+
+miniconda_ver=os.getenv("miniconda_ver") or "23.5.2-hafs"
+load(pathJoin("miniconda", miniconda_ver))
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+-- stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+-- load(pathJoin("stack-python", stack_python_ver))
+
+local ufs_modules = {
+ {["jasper"] = "2.0.32"},
+ {["zlib"] = "1.2.13"},
+ {["libpng"] = "1.6.37"},
+ {["hdf5"] = "1.14.0"},
+ {["netcdf-c"] = "4.9.2"},
+ {["netcdf-fortran"] = "4.6.0"},
+ {["parallelio"] = "2.5.10"},
+ {["esmf"] = "8.4.2"},
+ {["fms"] = "2023.01"},
+ {["bacio"] = "2.4.1"},
+ {["crtm"] = "2.4.0"},
+ {["g2"] = "3.4.5"},
+ {["g2tmpl"] = "1.10.2"},
+ {["ip"] = "3.3.3"},
+ {["nemsio"] = "2.5.4"},
+ {["sp"] = "2.3.3"},
+ {["w3emc"] = "2.9.2"},
+ {["w3nco"] = "2.4.1"},
+ {["gftl-shared"] = "1.5.0"},
+-- no yafyaml
+ {["mapl"] = "2.35.2-esmf-8.4.2"},
+ {["bufr"] = "11.7.0"},
+ {["sfcio"] = "1.4.1"},
+ {["sigio"] = "2.3.2"},
+-- no szip
+ {["wrf-io"] = "1.2.0"},
+ {["prod-util"] = "1.2.2"},
+ {["grib-util"] = "1.2.3"},
+ {["wgrib2"] = "2.0.8"},
+-- no gempak
+ {["nco"] = "5.0.6"},
+ {["cdo"] = "2.0.5"},
+}
+
+for i = 1, #ufs_modules do
+ for name, default_version in pairs(ufs_modules[i]) do
+ local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+ load(pathJoin(name, os.getenv(env_version_name) or default_version))
+ end
+end
+
+load("rocoto")
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CMAKE_C_COMPILER", "cc")
+setenv("CMAKE_CXX_COMPILER", "CC")
+setenv("CMAKE_Fortran_COMPILER", "ftn")
+setenv("CMAKE_Platform", "gaea_c5.intel")
diff --git a/modulefiles/hafs.hera.lua b/modulefiles/hafs.hera.lua
index d808b6501..4bc866b73 100644
--- a/modulefiles/hafs.hera.lua
+++ b/modulefiles/hafs.hera.lua
@@ -6,122 +6,65 @@ prepend_path("MODULEPATH", "/contrib/sutils/modulefiles")
load("sutils")
load("hpss")
-cmake_ver=os.getenv("cmake_ver") or "3.20.1"
-load(pathJoin("cmake", cmake_ver))
-
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles")
-load(pathJoin("python","wcoss2_env"))
-
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
-hpc_ver=os.getenv("hpc_ver") or "1.2.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2"
-load(pathJoin("hpc-impi", hpc_impi_ver))
-
-jasper_ver=os.getenv("jasper_ver") or "2.0.25"
-load(pathJoin("jasper", jasper_ver))
-
-zlib_ver=os.getenv("zlib_ver") or "1.2.11"
-load(pathJoin("zlib", zlib_ver))
-
-zstd_ver=os.getenv("zstd_ver") or "1.5.0"
-load(pathJoin("zstd", zstd_ver))
-
-libpng_ver=os.getenv("libpng_ver") or "1.6.37"
-load(pathJoin("libpng", libpng_ver))
-
-hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
-load(pathJoin("hdf5", hdf5_ver))
-
-netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
-load(pathJoin("netcdf", netcdf_ver))
-
-pio_ver=os.getenv("pio_ver") or "2.5.7"
-load(pathJoin("pio", pio_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.3.0b09"
-load(pathJoin("esmf", esmf_ver))
-
-fms_ver=os.getenv("fms_ver") or "2022.04"
-load(pathJoin("fms",fms_ver))
-
-bacio_ver=os.getenv("bacio_ver") or "2.4.1"
-load(pathJoin("bacio", bacio_ver))
-
-crtm_ver=os.getenv("crtm_ver") or "2.4.0"
-load(pathJoin("crtm", crtm_ver))
-
-g2_ver=os.getenv("g2_ver") or "3.4.5"
-load(pathJoin("g2", g2_ver))
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
-g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2"
-load(pathJoin("g2tmpl", g2tmpl_ver))
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
-ip_ver=os.getenv("ip_ver") or "3.3.3"
-load(pathJoin("ip", ip_ver))
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
-nemsio_ver_ver=os.getenv("nemsio_ver") or "2.5.4"
-load(pathJoin("nemsio", nemsio_ver))
-
-sp_ver=os.getenv("sp_ver") or "2.3.3"
-load(pathJoin("sp", sp_ver))
-
-w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
-load(pathJoin("w3emc", w3emc_ver))
-
-w3nco_ver=os.getenv("w3nco_ver") or "2.4.1"
-load(pathJoin("w3nco", w3nco_ver))
-
-gftl_shared_ver=os.getenv("gftl_shared_ver") or "v1.5.0"
-load(pathJoin("gftl-shared", gftl_shared_ver))
-
-yafyaml_ver=os.getenv("yafyaml_ver") or "v0.5.1"
-load(pathJoin("yafyaml", yafyaml_ver))
-
-mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09"
-load(pathJoin("mapl", mapl_ver))
-
-bufr_ver=os.getenv("bufr_ver") or "11.4.0"
-load(pathJoin("bufr", bufr_ver))
-
-sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
-load(pathJoin("sfcio", sfcio_ver))
-
-sigio_ver=os.getenv("sigio_ver") or "2.3.2"
-load(pathJoin("sigio", sigio_ver))
-
-szip_ver=os.getenv("szip_ver") or "2.1.1"
-load(pathJoin("szip", szip_ver))
-
-wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
-load(pathJoin("wrf_io", wrf_io_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.4"
-load(pathJoin("grib_util", grib_util_ver))
-
-wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8"
-load(pathJoin("wgrib2", wgrib2_ver))
-setenv("WGRIB2", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/impi-2018.0.4/wgrib2/2.0.8/bin/wgrib2")
-
-gempak_ver=os.getenv("gempak_ver") or "7.4.2"
-load(pathJoin("gempak", gempak_ver))
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
--- nco_ver=os.getenv("nco_ver") or "5.0.6"
-nco_ver=os.getenv("nco_ver") or "4.9.3"
-load(pathJoin("nco", nco_ver))
+local hafs_modules = {
+ {["jasper"] = "2.0.32"},
+ {["zlib"] = "1.2.13"},
+ {["zstd"] = "1.5.2"},
+ {["libpng"] = "1.6.37"},
+ {["hdf5"] = "1.14.0"},
+ {["netcdf-c"] = "4.9.2"},
+ {["netcdf-fortran"] = "4.6.0"},
+ {["parallelio"] = "2.5.10"},
+ {["esmf"] = "8.4.2"},
+ {["fms"] = "2023.01"},
+ {["bacio"] = "2.4.1"},
+ {["crtm"] = "2.4.0"},
+ {["g2"] = "3.4.5"},
+ {["g2tmpl"] = "1.10.2"},
+ {["ip"] = "4.3.0"},
+ {["nemsio"] = "2.5.4"},
+ {["sp"] = "2.3.3"},
+ {["w3emc"] = "2.10.0"},
+ {["w3nco"] = "2.4.1"},
+ {["gftl-shared"] = "1.5.0"},
+ {["yafyaml"] = "0.5.1"},
+ {["mapl"] = "2.35.2-esmf-8.4.2"},
+ {["bufr"] = "12.0.0"},
+ {["sfcio"] = "1.4.1"},
+ {["sigio"] = "2.3.2"},
+ {["szip"] = "2.1"},
+ {["wrf-io"] = "1.2.0"},
+ {["prod_util"] = "1.2.2"},
+ {["grib-util"] = "1.3.0"},
+ {["wgrib2"] = "2.0.8"},
+ {["gempak"] = "7.4.2"},
+ {["nco"] = "5.0.6"},
+ {["cdo"] = "2.0.5"},
+ {["rocoto"] = "1.3.3"},
+ {["scotch"] = "7.0.4"},
+}
+
+for i = 1, #hafs_modules do
+ for name, default_version in pairs(hafs_modules[i]) do
+ local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+ load(pathJoin(name, os.getenv(env_version_name) or default_version))
+ end
+end
-cdo_ver=os.getenv("cdo_ver") or "1.9.8"
-load(pathJoin("cdo", cdo_ver))
-rocoto_ver=os.getenv("rocoto_ver") or "1.3.3"
-load(pathJoin("rocoto", rocoto_ver))
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles")
+load(pathJoin("python","wcoss2_env"))
setenv("CMAKE_C_COMPILER", "mpiicc")
setenv("CMAKE_CXX_COMPILER", "mpiicpc")
diff --git a/parm/WaterInTheWest.conf b/parm/WaterInTheWest.conf
index e0adac941..5eea21450 100644
--- a/parm/WaterInTheWest.conf
+++ b/parm/WaterInTheWest.conf
@@ -69,9 +69,9 @@ jend_nest=1294
regional_esg=no
[forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
# For the global domain if it is a global or global-nesting experiment
glob_k_split=1
@@ -111,7 +111,8 @@ npz=64
is_moving_nest=.false.
quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
+write_tasks_per_group=360
output_history=.true.
output_grid=regional_latlon,regional_latlon
@@ -160,8 +161,8 @@ lightning_threat=.true.
satpost=.false.
post_gridspecs=auto,auto
trak_gridspecs={post_gridspecs}
-postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
-postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
+# postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
+# postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
[atm_post_ens]
satpost_ens=.false.
@@ -169,4 +170,5 @@ post_gridspecs_ens=auto,auto
trak_gridspecs_ens={post_gridspecs}
[rocotostr]
-FORECAST_RESOURCES=FORECAST_RESOURCES_2460PE
+FORECAST_RESOURCES=FORECAST_RESOURCES_3060PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES
\ No newline at end of file
diff --git a/parm/WaterInTheWestC192.conf b/parm/WaterInTheWestC192.conf
index 2fa2f02b5..ce522f99a 100644
--- a/parm/WaterInTheWestC192.conf
+++ b/parm/WaterInTheWestC192.conf
@@ -69,9 +69,9 @@ jend_nest=322
regional_esg=no
[forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
# For the global domain if it is a global or global-nesting experiment
glob_k_split=1
@@ -113,7 +113,7 @@ npz=64
is_moving_nest=.false.
quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
output_history=.true.
output_grid=regional_latlon,regional_latlon
@@ -174,3 +174,4 @@ trak_gridspecs_ens={post_gridspecs}
[rocotostr]
FORECAST_RESOURCES=FORECAST_RESOURCES_600PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES
diff --git a/parm/WaterInTheWestC384.conf b/parm/WaterInTheWestC384.conf
index de7feb798..947860e7c 100644
--- a/parm/WaterInTheWestC384.conf
+++ b/parm/WaterInTheWestC384.conf
@@ -69,9 +69,9 @@ jend_nest=648
regional_esg=no
[forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
# For the global domain if it is a global or global-nesting experiment
glob_k_split=1
@@ -111,7 +111,8 @@ npz=64
is_moving_nest=.false.
quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
+write_tasks_per_group=120
output_history=.true.
output_grid=regional_latlon,regional_latlon
@@ -171,4 +172,5 @@ post_gridspecs_ens=auto,auto
trak_gridspecs_ens={post_gridspecs}
[rocotostr]
-FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE
+FORECAST_RESOURCES=FORECAST_RESOURCES_1440PE
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES
diff --git a/parm/WaterInTheWestC384NoNest.conf b/parm/WaterInTheWestC384NoNest.conf
new file mode 100644
index 000000000..289bbb00e
--- /dev/null
+++ b/parm/WaterInTheWestC384NoNest.conf
@@ -0,0 +1,15 @@
+# This is a UNIX conf file that overrrides WaterInTheWestC384.conf to run
+# the HAFS stretched globe configuration with no nest.
+# This file must be after that one in the run_hafs.py command line.
+#
+
+[grid]
+gtype=stretch ;; grid type: uniform, stretch, nest, or regional
+nest_grids=0
+
+[forecast]
+glob_layoutx=10
+glob_layouty=10
+
+[rocotostr]
+FORECAST_RESOURCES=FORECAST_RESOURCES_840PE
diff --git a/parm/WaterInTheWestC384Uniform.conf b/parm/WaterInTheWestC384Uniform.conf
new file mode 100644
index 000000000..6076b8d8e
--- /dev/null
+++ b/parm/WaterInTheWestC384Uniform.conf
@@ -0,0 +1,14 @@
+# This is a UNIX conf file that overrrides WaterInTheWestC384.conf to run
+# the HAFS uniform C384 global configuration (with no nest)
+# This file must be after that one in the run_hafs.py command line.
+#
+[grid]
+gtype=uniform ;; grid type: uniform, stretch, nest, or regional
+nest_grids=0
+
+[forecast]
+glob_layoutx=10
+glob_layouty=10
+
+[rocotostr]
+FORECAST_RESOURCES=FORECAST_RESOURCES_840PE
diff --git a/parm/WaterInTheWestC96.conf b/parm/WaterInTheWestC96.conf
index cf5a1b6bc..916db6a4e 100644
--- a/parm/WaterInTheWestC96.conf
+++ b/parm/WaterInTheWestC96.conf
@@ -69,9 +69,9 @@ jend_nest=160
regional_esg=no
[forecast]
-ccpp_suite_regional=FV3_RRFSv2c3
-ccpp_suite_glob=FV3_RRFSv2c3
-ccpp_suite_nest=FV3_RRFSv2c3
+ccpp_suite_regional=FV3_global_nest_v0
+ccpp_suite_glob=FV3_global_nest_v0
+ccpp_suite_nest=FV3_global_nest_v0
# For the global domain if it is a global or global-nesting experiment
glob_k_split=1
@@ -113,7 +113,7 @@ npz=64
is_moving_nest=.false.
quilting=.true.
-write_dopost=.false.
+write_dopost=.true.
output_history=.true.
output_grid=regional_latlon,regional_latlon
@@ -163,8 +163,8 @@ lightning_threat=.true.
satpost=.false.
post_gridspecs=auto,auto
trak_gridspecs={post_gridspecs}
-postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
-postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
+# postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt
+# postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt
[atm_post_ens]
satpost_ens=.false.
@@ -173,4 +173,4 @@ trak_gridspecs_ens={post_gridspecs}
[rocotostr]
FORECAST_RESOURCES=FORECAST_RESOURCES_240PE
-ATM_POST_RESOURCES=ATM_POST_SMALL_RESOURCES
+ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES
\ No newline at end of file
diff --git a/parm/forecast/globnest/input.nml.nonest.tmp b/parm/forecast/globnest/input.nml.nonest.tmp
new file mode 100644
index 000000000..27bec3441
--- /dev/null
+++ b/parm/forecast/globnest/input.nml.nonest.tmp
@@ -0,0 +1,421 @@
+&atmos_model_nml
+ blocksize = @[blocksize]
+ chksum_debug = .false.
+ dycore_only = .false.
+ avg_max_length = 10800.
+ ccpp_suite = '@[ccpp_suite_nml]'
+ ignore_rst_cksum = .true.
+/
+
+&diag_manager_nml
+ prepend_date = .false.
+/
+
+&mpp_io_nml
+ header_buffer_val = 16384,
+ global_field_on_root_pe = .true.,
+ io_clocks_on = .false.,
+ shuffle = 0,
+ deflate_level = @[deflate_level],
+ cf_compliance = .false.
+/
+
+&fms_io_nml
+ checksum_required = .false.
+ max_files_r = 1000,
+ max_files_w = 1000,
+/
+
+&fms2_io_nml
+ netcdf_default_format="netcdf4"
+/
+
+&fms_nml
+ clock_grain = 'ROUTINE',
+ domains_stack_size = 30000000,
+ print_memory_usage = .false.
+/
+
+&fv_grid_nml
+ !grid_file = 'INPUT/grid_spec.nc'
+/
+
+&fv_core_nml
+ ignore_rst_cksum = .true.
+ layout = @[layoutx_nml],@[layouty_nml]
+ io_layout = @[io_layoutx_nml],@[io_layouty_nml]
+ npx = @[npx_nml]
+ npy = @[npy_nml]
+ ntiles = 6
+ npz = @[npz]
+ !grid_type = -1
+ make_nh = .false.
+ fv_debug = .false.
+ range_warn = .true.
+ reset_eta = .false.
+ upoff = 2
+ n_sponge = 24
+ nudge_qv = .false.
+ nudge_dz = .false.
+ tau = 5.
+ rf_cutoff = 30.e2
+ sg_cutoff = -1.
+ rf_fast = .false.
+ d2_bg_k1 = 0.16
+ d2_bg_k2 = 0.05
+ kord_tm = -9
+ kord_mt = 9
+ kord_wz = 9
+ kord_tr = 9
+ hydrostatic = .false.
+ phys_hydrostatic = .false.
+ use_hydro_pressure = .false.
+ beta = 0.
+ a_imp = 1.
+ p_fac = 0.1
+ k_split = @[k_split_nml]
+ n_split = @[n_split_nml]
+ nwat = 6
+ na_init = @[na_init]
+ d_ext = 0.0
+ dnats = @[dnats]
+ fv_sg_adj = 300
+ d2_bg = 0.
+ nord = 2
+ dddmp = 0.1
+ d4_bg = 0.15
+ vtdm4 = 0.04
+ delt_max = 0.002
+ dz_min = 2.0
+ ke_bg = 0.
+ do_vort_damp = .true.
+ external_ic = @[external_ic]
+ external_eta = .true.
+ gfs_phil = .false.
+ nggps_ic = @[nggps_ic]
+ mountain = @[mountain]
+ ncep_ic = .false.
+ d_con = 1.0
+ hord_mt = 5
+ hord_vt = 5
+ hord_tm = 5
+ hord_dp = -5
+ hord_tr = -5
+ adjust_dry_mass = .false.
+ consv_te = 1.
+ do_sat_adj = @[do_sat_adj]
+ consv_am = .false.
+ fill = .true.
+ dwind_2d = .false.
+ print_freq = 3
+ warm_start = @[warm_start]
+ no_dycore = .false.
+ z_tracer = .true.
+ agrid_vel_rst = .true.
+ read_increment = .false.
+ res_latlon_dynamics = "fv3_increment.nc"
+ write_3d_diags = .true.
+
+ do_schmidt = .true.
+ target_lat = @[target_lat]
+ target_lon = @[target_lon]
+ stretch_fac = @[stretch_fac]
+
+ full_zs_filter = @[full_zs_filter_nml]
+ n_zs_filter = @[n_zs_filter_nml]
+ nord_zs_filter = 4
+/
+
+
+&surf_map_nml
+ zero_ocean = .false.
+ cd4 = 0.15
+ cd2 = -1
+ n_del2_strong = 0
+ n_del2_weak = @[n_del2_weak_nml]
+ n_del4 = 2
+ max_slope = @[max_slope_nml]
+ peak_fac = 1.
+/
+
+&external_ic_nml
+ filtered_terrain = .true.
+ levp = @[levp]
+ gfs_dwinds = .true.
+ checker_tr = .false.
+ nt_checker = 0
+/
+
+&gfs_physics_nml
+ lightning_threat = @[lightning_threat]
+ fhzero = 3.
+ ldiag3d = .false.
+ lradar = .true.
+ avg_max_length = 10800.
+ h2o_phys = .true.
+ fhcyc = 0.
+ use_ufo = .true.
+ pre_rad = .false.
+ imp_physics = @[imp_physics]
+ iovr = @[iovr]
+ ltaerosol = @[ltaerosol]
+ ttendlim = -999
+ dt_inner = @[dt_inner]
+ sedi_semi = .true.
+ decfl = 10
+ pdfcld = .false.
+ min_lakeice = 0.15
+ min_seaice = 0.15
+ use_cice_alb = .false
+ fhswr = @[fhswr]
+ fhlwr = @[fhlwr]
+ ialb = @[ialb]
+ iems = @[iems]
+ iaer = @[iaer]
+ ico2 = 2
+ isubc_sw = 2
+ isubc_lw = 2
+ isol = 2
+ lwhtr = .true.
+ swhtr = .true.
+ cnvgwd = .true.
+ cal_pre = .false.
+ redrag = .true.
+ dspheat = .true.
+ hybedmf = .false.
+ hurr_pbl = .true. ! HWRF moninedmf
+ moninq_fac = -1.0 ! HWRF_moninedmf
+ do_mynnedmf = @[do_mynnedmf]
+ do_mynnsfclay = @[do_mynnsfclay]
+ bl_mynn_tkeadvect = @[bl_mynn_tkeadvect]
+ satmedmf = @[satmedmf]
+ isatmedmf = 1
+ lseaspray = @[lseaspray]
+ rlmx = @[rlmx_nml]
+ elmx = @[elmx_nml]
+ sfc_rlm = @[sfc_rlm_nml]
+ tc_pbl = @[tc_pbl_nml]
+ random_clds = .false.
+ trans_trac = .true.
+ cnvcld = .true.
+ shal_cnv = @[shal_cnv_nml] !Shallow convection
+ do_deep = @[do_deep_nml]
+ imfshalcnv = @[imfshalcnv]
+ imfdeepcnv = @[imfdeepcnv]
+ clam_deep = @[clam_deep]
+ betal_deep = 0.05
+ betas_deep = 0.05
+ cdmbgwd = @[cdmbgwd]
+ sfc_z0_type = 6
+ prslrd0 = 0.
+ ivegsrc = 1
+ isot = 1
+ lsoil = 4
+ lsm = @[lsm]
+ iopt_dveg = 4
+ iopt_crs = 2
+ iopt_btr = 1
+ iopt_run = 1
+ iopt_sfc = @[iopt_sfc]
+ iopt_frz = 1
+ iopt_inf = 1
+ iopt_rad = 3
+ iopt_alb = 1
+ iopt_snf = 4
+ iopt_tbot = 2
+ iopt_stc = 3
+ iopt_trs = 2
+ lsoil_lsm = 4
+ debug = .false.
+ oz_phys = .false.
+ oz_phys_2015 = .true.
+ nst_anl = .true.
+ nstf_name = @[nstf_n1],@[nstf_n2],@[nstf_n3],@[nstf_n4],@[nstf_n5]
+ cplflx = @[cplflx]
+ cplice = .false.
+ cplocn2atm = @[cplocn2atm]
+ cplwav = @[cplwav]
+ cplwav2atm = @[cplwav2atm]
+ cpl_imp_mrg = @[merge_import]
+ psautco = 0.0008, 0.0005
+ prautco = 0.00015, 0.00015
+ iau_delthrs = 6
+ iaufhrs = 30
+ iau_inc_files = ''
+ iau_drymassfixer = .false.
+ lgfdlmprad = @[lgfdlmprad]
+ effr_in = .true.
+ do_sppt = @[do_sppt]
+ do_shum = @[do_shum]
+ do_skeb = @[do_skeb]
+ lndp_type = 0
+ n_var_lndp = 0
+ do_ca = .false.
+ ca_sgs = .false.
+ nca = 1
+ ncells = 5
+ nlives = 12
+ nseed = 1
+ nfracseed = 0.5
+ ca_trigger = .false.
+ ca_entr = .false.
+ ca_closure = .false.
+ ca_global = .false.
+ nca_g = 1
+ ncells_g = 1
+ nlives_g = 100
+ nseed_g = 100
+ ca_smooth = .false.
+ nspinup = 1
+ iseed_ca = 1
+ nsmooth = 100
+ ca_amplitude = 0.35
+ print_diff_pgr = .true.
+ do_ugwp = .false.
+ do_tofd = .false.
+ gwd_opt = @[gwd_opt]
+ do_ugwp_v0 = @[do_ugwp_v0]
+ do_ugwp_v1 = @[do_ugwp_v1]
+ do_ugwp_v0_orog_only = @[do_ugwp_v0_orog_only]
+ do_ugwp_v0_nst_only = @[do_ugwp_v0_nst_only]
+ do_ugwp_v1_w_gsldrag = @[do_ugwp_v1_w_gsldrag]
+ do_ugwp_v1_orog_only = @[do_ugwp_v1_orog_only]
+ do_gsl_drag_ls_bl = @[do_gsl_drag_ls_bl]
+ do_gsl_drag_ss = @[do_gsl_drag_ss]
+ do_gsl_drag_tofd = @[do_gsl_drag_tofd]
+/
+
+&gfdl_cloud_microphysics_nml
+ sedi_transport = .true.
+ do_sedi_heat = .false.
+ rad_snow = .true.
+ rad_graupel = .true.
+ rad_rain = .true.
+ const_vi = .false.
+ const_vs = .false.
+ const_vg = .false.
+ const_vr = .false.
+ vi_max = 1.
+ vs_max = 2.
+ vg_max = 12.
+ vr_max = 12.
+ qi_lim = 1.
+ prog_ccn = .false.
+ do_qa = .true.
+ fast_sat_adj = .true.
+ tau_l2v = 180.
+ tau_v2l = 90.
+ tau_g2v = 900.
+ rthresh = 10.e-6 ! This is a key parameter for cloud water
+ dw_land = 0.16
+ dw_ocean = 0.10
+ ql_gen = 1.0e-3
+ ql_mlt = 1.0e-3
+ qi0_crt = 8.0E-5
+ qs0_crt = 1.0e-3
+ tau_i2s = 1000.
+ c_psaci = 0.05
+ c_pgacs = 0.01
+ rh_inc = 0.30
+ rh_inr = 0.30
+ rh_ins = 0.30
+ ccn_l = 300.
+ ccn_o = 100.
+ c_paut = 0.5
+ c_cracw = 0.8
+ use_ppm = .false.
+ use_ccn = .true.
+ mono_prof = .true.
+ z_slope_liq = .true.
+ z_slope_ice = .true.
+ de_ice = .false.
+ fix_negative = .true.
+ icloud_f = 1
+ mp_time = 90.
+/
+
+&interpolator_nml
+ interp_method = 'conserve_great_circle'
+/
+
+&namsfc
+ FNGLAC = "global_glacier.2x2.grb",
+ FNMXIC = "global_maxice.2x2.grb",
+ FNTSFC = "RTGSST.1982.2012.monthly.clim.grb",
+ FNSNOC = "global_snoclim.1.875.grb",
+ FNZORC = "igbp"
+ !FNZORC = "global_zorclim.1x1.grb",
+ FNALBC = "global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb",
+ FNALBC2 = "global_albedo4.1x1.grb",
+ FNAISC = "CFSR.SEAICE.1982.2012.monthly.clim.grb",
+ FNTG3C = "global_tg3clim.2.6x1.5.grb",
+ FNVEGC = "global_vegfrac.0.144.decpercent.grb",
+ FNVETC = "global_vegtype.igbp.t1534.3072.1536.rg.grb",
+ FNSOTC = "global_soiltype.statsgo.t1534.3072.1536.rg.grb",
+ FNSMCC = "global_soilmgldas.t1534.3072.1536.grb",
+ FNMSKH = "seaice_newland.grb",
+ FNTSFA = "",
+ FNACNA = "",
+ FNSNOA = "",
+ FNVMNC = "global_shdmin.0.144x0.144.grb",
+ FNVMXC = "global_shdmax.0.144x0.144.grb",
+ FNSLPC = "global_slope.1x1.grb",
+ FNABSC = "global_mxsnoalb.uariz.t1534.3072.1536.rg.grb",
+ LDEBUG = .true.,
+ FSMCL(2) = 99999
+ FSMCL(3) = 99999
+ FSMCL(4) = 99999
+ FTSFS = 90
+ FAISS = 99999
+ FSNOL = 99999
+ FSICL = 99999
+ FTSFL = 99999
+ FAISL = 99999
+ FVETL = 99999,
+ FSOTL = 99999,
+ FvmnL = 99999,
+ FvmxL = 99999,
+ FSLPL = 99999,
+ FABSL = 99999,
+ FSNOS = 99999,
+ FSICS = 99999,
+/
+
+&nam_stochy
+ skebnorm = 1
+ skeb_npass = 30
+ skeb_vdof = 5
+ skeb = 0.6,-999,-999,-999,-999
+ skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7
+ skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3
+ shum = 0.003
+ shum_tau = 21600
+ shum_lscale = 500000
+ sppt = 0.4,0.2,0.1,0.04,0.02
+ sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7
+ sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3
+ sppt_logit = .true.
+ sppt_sfclimit = .true.
+ use_zmtnblck = .true.
+ iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5]
+ iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5]
+ iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5]
+/
+
+&nam_sfcperts
+/
+
+&cires_ugwp_nml
+ knob_ugwp_solver = 2
+ knob_ugwp_source = 1,1,0,0
+ knob_ugwp_wvspec = 1,25,25,25
+ knob_ugwp_azdir = 2,4,4,4
+ knob_ugwp_stoch = 0,0,0,0
+ knob_ugwp_effac = 1,1,1,1
+ knob_ugwp_doaxyz = 1
+ knob_ugwp_doheat = 1
+ knob_ugwp_dokdis = 1
+ knob_ugwp_ndx4lh = 1
+ knob_ugwp_version = 0
+ launch_level = 25
+/
diff --git a/parm/forecast/globnest/nems.configure.atmonly b/parm/forecast/globnest/ufs.configure.atmonly
similarity index 100%
rename from parm/forecast/globnest/nems.configure.atmonly
rename to parm/forecast/globnest/ufs.configure.atmonly
diff --git a/parm/forecast/regional/nems.configure.atmonly b/parm/forecast/regional/ufs.configure.atmonly
similarity index 100%
rename from parm/forecast/regional/nems.configure.atmonly
rename to parm/forecast/regional/ufs.configure.atmonly
diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/ufs.configure.cdeps.tmp
similarity index 100%
rename from parm/forecast/regional/nems.configure.cdeps.tmp
rename to parm/forecast/regional/ufs.configure.cdeps.tmp
diff --git a/parm/forecast/regional/nems.configure.cpl.tmp b/parm/forecast/regional/ufs.configure.cpl.tmp
similarity index 100%
rename from parm/forecast/regional/nems.configure.cpl.tmp
rename to parm/forecast/regional/ufs.configure.cpl.tmp
diff --git a/parm/forecast/regional/nems.configure.mom6.tmp b/parm/forecast/regional/ufs.configure.mom6.tmp
similarity index 100%
rename from parm/forecast/regional/nems.configure.mom6.tmp
rename to parm/forecast/regional/ufs.configure.mom6.tmp
diff --git a/parm/hafs.conf b/parm/hafs.conf
index c5bd548b8..355996c63 100644
--- a/parm/hafs.conf
+++ b/parm/hafs.conf
@@ -30,6 +30,8 @@ input_catalog=fcst_{GFSVER}
docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, or NONE)
datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE)
+pygraf_can_run=yes
+
## Configure file and directory paths
[dir]
HOMEhafs={CDSAVE}/{EXPT}
diff --git a/parm/system.conf.gaea b/parm/system.conf.gaea
new file mode 100644
index 000000000..bb289679d
--- /dev/null
+++ b/parm/system.conf.gaea
@@ -0,0 +1,53 @@
+## This is the system-specific configuration file for Hera
+[config]
+## Project disk area
+disk_project=esrl
+## Project hpss tape area
+tape_project=wrfruc
+## CPU account name for submitting jobs to the batch system.
+cpu_account=gsl-glo
+## Archive path
+archive=hpss:/BMC/{tape_project}/2year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar
+
+pygraf_can_run=no
+
+[dir]
+gaea_input=/lustre/f2/dev/Samuel.Trahan/hafs/hafs-input
+hafs_work=/lustre/f2/dev/{ENV[USER]}/hafs/work
+
+## Save directory. Make sure you edit this.
+CDSAVE={hafs_work}
+## Non-scrubbed directory for track files, etc. Make sure you edit this.
+CDNOSCRUB={hafs_work}/noscrub/hafstrak
+## Scrubbed directory for large work files. Make sure you edit this.
+CDSCRUB=/lustre/f2/scratch/{ENV[USER]}
+## Syndat directory for finding which cycles to run
+syndat={gaea_input}/SYNDAT-PLUS
+#oldcom={ENV[COMOLD]}
+COMOLD={oldcom}
+COMIN={COMhafs}
+COMOUT={COMhafs}
+COMINnhc={ENV[DCOMROOT|-/dcom]}/nhc/atcf/ncep
+COMINjtwc={ENV[DCOMROOT|-/dcom]}/{ENV[PDY]}/wtxtbul/storm_data
+COMgfs={gaea_input}/COMGFSv16
+COMINobs={COMgfs}
+COMINgfs={COMgfs}
+COMINgdas={COMgfs}
+COMINarch={COMgfs}/syndat
+COMrtofs={gaea_input}/COMRTOFSv2
+COMINrtofs={COMrtofs}
+COMINmsg={COMINgfs}
+COMINhafs={COMINgfs}
+DATMdir={hafs_work}/noscrub/DATM
+DOCNdir={hafs_work}/noscrub/DOCN
+## A-Deck directory for graphics
+ADECKhafs={gaea_input}/aid
+## B-Deck directory for graphics
+BDECKhafs={gaea_input}/btk
+## cartopyDataDir directory for graphics
+cartopyDataDir={gaea_input}/cartopy
+
+[holdvars]
+WHERE_AM_I=gaea_c5
+WHICH_JET=none
+
diff --git a/parm/system.conf.hera b/parm/system.conf.hera
index 0f0c97e61..7e57b1f9f 100644
--- a/parm/system.conf.hera
+++ b/parm/system.conf.hera
@@ -24,12 +24,14 @@ COMIN={COMhafs}
COMOUT={COMhafs}
COMINnhc={ENV[DCOMROOT|-/dcom]}/nhc/atcf/ncep
COMINjtwc={ENV[DCOMROOT|-/dcom]}/{ENV[PDY]}/wtxtbul/storm_data
-COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16
+# COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16
+COMgfs=/scratch2/BMC/wrfruc/Samuel.Trahan/westwater/hafs-input/COMGFSv16
COMINobs={COMgfs}
COMINgfs={COMgfs}
COMINgdas={COMgfs}
COMINarch={COMgfs}/syndat
-COMrtofs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMRTOFSv2
+# COMrtofs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMRTOFSv2
+COMrtofs=/scratch2/BMC/wrfruc/Samuel.Trahan/westwater/hafs-input/COMRTOFSv2
COMINrtofs={COMrtofs}
COMINmsg={COMINgfs}
COMINhafs={COMINgfs}
diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh
index f28182fd7..93ba7d5f5 100755
--- a/rocoto/WaterInTheWest.sh
+++ b/rocoto/WaterInTheWest.sh
@@ -10,9 +10,10 @@ EXPT=$(basename ${HOMEhafs})
SUBEXPT=$EXPT
conf=../parm/WaterInTheWest.conf
opts="-t -f"
+
scrubopt="config.scrub_work=no config.scrub_com=no"
# 3DEnVar with GDAS ensembles
- ./run_hafs.py ${opts} 2020082506-2020082512 00L HISTORY \
+ ./run_hafs.py ${opts} 2023022400 00L HISTORY \
config.EXPT=${EXPT} config.SUBEXPT=${SUBEXPT} \
- config.NHRS=12 ${scrubopt} \
- $conf
+ config.NHRS=24 ${scrubopt} \
+ $conf config.run_pygraf=yes
diff --git a/rocoto/sites/all.ent b/rocoto/sites/all.ent
index e2e3b6680..cafba9c6a 100644
--- a/rocoto/sites/all.ent
+++ b/rocoto/sites/all.ent
@@ -9,3 +9,4 @@
+
diff --git a/rocoto/sites/gaea_c5.ent b/rocoto/sites/gaea_c5.ent
new file mode 100644
index 000000000..474d7f4c4
--- /dev/null
+++ b/rocoto/sites/gaea_c5.ent
@@ -0,0 +1,85 @@
+
+
+
+
+
+
+
+
+
+
+
+ --clusters=c5--exclusive">
+
+
+
+ 128&C5_EXTRA;">
+ 128&C5_EXTRA;">
+
+
+
+ 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:00">
+ 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00">
+ 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00">
+ 1:ppn=128TOTAL_TASKS18NCTSK6OMP_THREADS601:30:00">
+ 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:00">
+ 3:ppn=60:tpp=1TOTAL_TASKS180NCTSK60OMP_THREADS100:15:00">
+ 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS101:30:00">
+ 3:ppn=60:tpp=1TOTAL_TASKS180NCTSK60OMP_THREADS100:30:00">
+ 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS101:30:00">
+ 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00">
+ 1:ppn=8:tpp=16TOTAL_TASKS8NCTSK8OMP_THREADS1600:30:00">
+ 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00">
+ 10:ppn=16:tpp=8TOTAL_TASKS160NCTSK16OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00">
+ 21:ppn=1:tpp=60TOTAL_TASKS21NCTSK1OMP_THREADS60-l place=vscatter:excl02:00:00">
+ 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS1-l place=vscatter:excl02:00:00">
+ 1:ppn=8:tpp=1TOTAL_TASKS8NCTSK8OMP_THREADS100:10:00">
+ 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00">
+
+ 05:59:00">
+ OMP_THREADS2">
+
+
+ 4:ppn=60:tpp=2TOTAL_TASKS240NCTSK60&FORECAST_EXTRA;">
+ 6:ppn=60:tpp=2TOTAL_TASKS360NCTSK60&FORECAST_EXTRA;">
+ 9:ppn=60:tpp=2TOTAL_TASKS540NCTSK60&FORECAST_EXTRA;">
+ 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60&FORECAST_EXTRA;">
+ 11:ppn=60:tpp=2TOTAL_TASKS660NCTSK60&FORECAST_EXTRA;">
+ 12:ppn=60:tpp=2TOTAL_TASKS720NCTSK60&FORECAST_EXTRA;">
+ 13:ppn=60:tpp=2TOTAL_TASKS780NCTSK60&FORECAST_EXTRA;">
+ 14:ppn=60:tpp=2TOTAL_TASKS840NCTSK60&FORECAST_EXTRA;">
+ 15:ppn=60:tpp=2TOTAL_TASKS900NCTSK60&FORECAST_EXTRA;">
+ 16:ppn=60:tpp=2TOTAL_TASKS960NCTSK60&FORECAST_EXTRA;">
+ 17:ppn=60:tpp=2TOTAL_TASKS1020NCTSK60&FORECAST_EXTRA;">
+ 18:ppn=60:tpp=2TOTAL_TASKS1080NCTSK60&FORECAST_EXTRA;">
+ 19:ppn=60:tpp=2TOTAL_TASKS1140NCTSK60&FORECAST_EXTRA;">
+ 20:ppn=60:tpp=2TOTAL_TASKS1200NCTSK60&FORECAST_EXTRA;">
+ 21:ppn=60:tpp=2TOTAL_TASKS1260NCTSK60&FORECAST_EXTRA;">
+ 22:ppn=60:tpp=2TOTAL_TASKS1320NCTSK60&FORECAST_EXTRA;">
+ 23:ppn=60:tpp=2TOTAL_TASKS1380NCTSK60&FORECAST_EXTRA;">
+ 24:ppn=60:tpp=2TOTAL_TASKS1440NCTSK60&FORECAST_EXTRA;">
+ 30:ppn=60:tpp=2TOTAL_TASKS1800NCTSK60&FORECAST_EXTRA;">
+ 36:ppn=60:tpp=2TOTAL_TASKS2160NCTSK60&FORECAST_EXTRA;">
+ 41:ppn=60:tpp=2TOTAL_TASKS2460NCTSK60&FORECAST_EXTRA;">
+ 43:ppn=60:tpp=2TOTAL_TASKS2580NCTSK60&FORECAST_EXTRA;">
+ 45:ppn=60:tpp=2TOTAL_TASKS2700NCTSK60&FORECAST_EXTRA;">
+ 46:ppn=60:tpp=2TOTAL_TASKS2760NCTSK60&FORECAST_EXTRA;">
+ 47:ppn=60:tpp=2TOTAL_TASKS2820NCTSK60&FORECAST_EXTRA;">
+ 49:ppn=60:tpp=2TOTAL_TASKS2940NCTSK60&FORECAST_EXTRA;">
+ 51:ppn=60:tpp=2TOTAL_TASKS3060NCTSK60&FORECAST_EXTRA;">
+
+ 90:ppn=20:tpp=2TOTAL_TASKS1800NCTSK20&FORECAST_EXTRA;">
+
+ 1:ppn=40:tpp=3TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00">
+ 1:ppn=40:tpp=3TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00">
+ 10:ppn=40:tpp=3TOTAL_TASKS400NCTSK40OMP_THREADS107:59:00">
+ 10:ppn=40:tpp=3TOTAL_TASKS400NCTSK40OMP_THREADS107:59:00">
+
+ 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00">
+ 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS100:30:00">
+ 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00">
+ 1:ppn=1:tpp=1TOTAL_TASKS2NCTSK1OMP_THREADS100:15:00">
+ 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00">
+
+ 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS107:59:00">
+ 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS107:59:00">
diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent
index eb7a20b51..8347dadbf 100644
--- a/rocoto/sites/hera.ent
+++ b/rocoto/sites/hera.ent
@@ -64,6 +64,7 @@
150:ppn=20:tpp=2TOTAL_TASKS3000NCTSK20&FORECAST_EXTRA;">
153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;">
+ 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00">
1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00">
6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS107:59:00">
8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20OMP_THREADS207:59:00">
diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent
index c31df72a8..9d834d0fc 100644
--- a/rocoto/sites/kjet.ent
+++ b/rocoto/sites/kjet.ent
@@ -60,6 +60,7 @@
147:ppn=20:tpp=2TOTAL_TASKS2940NCTSK20&FORECAST_EXTRA;">
153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;">
+ 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00">
3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00">
3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00">
3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00">
diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent
index 25400d95e..f09702e9b 100644
--- a/rocoto/sites/orion.ent
+++ b/rocoto/sites/orion.ent
@@ -63,6 +63,7 @@
150:ppn=20:tpp=2TOTAL_TASKS3000NCTSK20&FORECAST_EXTRA;">
153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;">
+ 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00">
1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00">
3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00">
8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20OMP_THREADS207:59:00">
diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent
index c1ac73c9d..cc5854875 100644
--- a/rocoto/sites/wcoss2.ent
+++ b/rocoto/sites/wcoss2.ent
@@ -61,6 +61,7 @@
49:ppn=60:tpp=2TOTAL_TASKS2940NCTSK60&FORECAST_EXTRA;">
51:ppn=60:tpp=2TOTAL_TASKS3060NCTSK60&FORECAST_EXTRA;">
+ 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00">
1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00">
1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00">
1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00">
diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent
index 545eb86a8..1f4ab1c6e 100644
--- a/rocoto/sites/xjet.ent
+++ b/rocoto/sites/xjet.ent
@@ -60,6 +60,7 @@
245:ppn=12:tpp=2TOTAL_TASKS2940NCTSK12&FORECAST_EXTRA;">
255:ppn=12:tpp=2TOTAL_TASKS3060NCTSK12&FORECAST_EXTRA;">
+ 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00">
5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00">
5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00">
5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00">
diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh
index 47989ab11..8c377c5d2 100755
--- a/scripts/exhafs_atm_post.sh
+++ b/scripts/exhafs_atm_post.sh
@@ -89,7 +89,7 @@ if [ "${POST_CLEANUP^^}" = "YES" ]; then
FHR3=$(printf "%03d" "$FHR")
# Loop for forecast hours
while [ $FHR -le $NHRS ]; do
- if [ ${gtype} = nest ]; then
+ if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
ngrids=$((${nest_grids} + 1))
else
ngrids=${nest_grids}
@@ -132,7 +132,7 @@ MM=$(echo $NEWDATE | cut -c5-6)
DD=$(echo $NEWDATE | cut -c7-8)
HH=$(echo $NEWDATE | cut -c9-10)
-if [ ${gtype} = nest ]; then
+if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
ngrids=$((${nest_grids} + 1))
else
ngrids=${nest_grids}
@@ -191,13 +191,13 @@ if [ ${write_dopost:-.false.} = .true. ]; then
# Wait for model output
n=1
-hurf=${INPdir}/HURPRS${neststr}.GrbF${FHR2}
+hurf=${INPdir}/HURPRS.GrbF${FHR2}${neststr}
while [ $n -le 360 ]; do
- if [ ! -s ${INPdir}/log.atm.f${FHR3} ] || [ ! -s ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ]; then
+ if [ ! -s ${INPdir}/log.atm.f${FHR3} ] || [ ! -s ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ]; then
echo "${INPdir}/log.atm.f${FHR3} not ready, sleep 10s"
sleep 10s
else
- echo "${INPdir}/log.atm.f${FHR3}, ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ready, continue"
+ echo "${INPdir}/log.atm.f${FHR3}, ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ready, continue"
sleep 1s
break
fi
@@ -243,9 +243,9 @@ cd ${DATA_POST}
# Note: Currently the inline post (write_dopost) does not support nesting configurations yet.
if [ ${write_dopost:-.false.} = .true. ]; then
-${NCP} -p ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ${grb2post}
+${NCP} -p ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ${grb2post}
if [ ${satpost} = .true. ]; then
- ${NCP} -p ${INPdir}/HURSAT${neststr}.GrbF${FHR2} ${sat_grb2post}
+ ${NCP} -p ${INPdir}/HURSAT.GrbF${FHR2}${neststr} ${sat_grb2post}
fi
else
diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh
index a673f03c3..689ee1069 100755
--- a/scripts/exhafs_forecast.sh
+++ b/scripts/exhafs_forecast.sh
@@ -347,7 +347,7 @@ if [ $gtype = regional ]; then
if [ $quilting = .true. ]; then
ATM_tasks=$(($ATM_tasks+$write_groups*$write_tasks_per_group))
fi
-elif [ $gtype = nest ]; then
+elif [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
ATM_tasks=$(( ${glob_layoutx} * ${glob_layouty} * 6 ))
for n in $(seq 1 ${nest_grids}); do
layoutx_tmp=$( echo ${layoutx} | cut -d , -f ${n} )
@@ -682,7 +682,7 @@ if [ ${imp_physics:-11} = 8 ]; then
${NLN} ${FIXam}/freezeH2O.dat ./
fi
-if [ $gtype = nest ]; then
+if [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
cd ./INPUT
@@ -736,17 +736,21 @@ fi
cd ..
-# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and nems.configure
+# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and ufs.configure
${NCP} ${PARMforecast}/diag_table.tmp .
if [ ${imp_physics:-11} = 8 ]; then
${NCP} ${PARMforecast}/field_table_thompson_aero ./field_table
else
${NCP} ${PARMforecast}/field_table .
fi
-${NCP} ${PARMforecast}/input.nml.tmp .
-${NCP} ${PARMforecast}/input_nest.nml.tmp .
+if [ $gtype = stretch ] || [ $gtype = uniform ]; then
+ ${NCP} ${PARMforecast}/input.nml.nonest.tmp input.nml.tmp
+else
+ ${NCP} ${PARMforecast}/input.nml.tmp .
+ ${NCP} ${PARMforecast}/input_nest.nml.tmp .
+fi
${NCP} ${PARMforecast}/model_configure.tmp .
-${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure
+${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure
# NoahMP table file
${NCP} ${PARMforecast}/noahmptable.tbl .
@@ -934,7 +938,7 @@ fi
cd ..
-# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and nems.configure
+# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and ufs.configure
${NCP} ${PARMforecast}/diag_table.tmp .
if [ ${imp_physics:-11} = 8 ]; then
${NCP} ${PARMforecast}/field_table_thompson ./field_table
@@ -950,16 +954,16 @@ ${NCP} ${PARMforecast}/noahmptable.tbl .
if [ ${ocean_model} = hycom ]; then
if [ ${run_ocean} = yes ] || [ ${run_wave} = yes ]; then
- ${NCP} ${PARMforecast}/nems.configure.cpl.tmp ./nems.configure.tmp
+ ${NCP} ${PARMforecast}/ufs.configure.cpl.tmp ./ufs.configure.tmp
else
- ${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure.tmp
+ ${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure.tmp
fi
elif [ ${ocean_model} = mom6 ]; then
if [ ${run_ocean} = yes ] || [ ${run_wave} = yes ]; then
- ${NCP} ${PARMforecast}/nems.configure.mom6.tmp ./nems.configure.tmp
+ ${NCP} ${PARMforecast}/ufs.configure.mom6.tmp ./ufs.configure.tmp
${NCP} ${PARMforecast}/data_table ./
else
- ${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure.tmp
+ ${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure.tmp
fi
else
echo "WARNING: unknow ocean model of ${ocean_model}"
@@ -990,7 +994,7 @@ sed -e "s/_EARTH_component_list_/${EARTH_component_list}/g" \
-e "/_mesh_atm_/d" \
-e "s/_mesh_wav_/ww3_mesh.nc/g" \
-e "s/_multigrid_/false/g" \
- nems.configure.tmp > nems.configure
+ ufs.configure.tmp > ufs.configure
ngrids=${nest_grids}
n=1
@@ -1230,10 +1234,10 @@ if [ ${run_datm} = yes ]; then
sed -i "s/_yearFirst_/$yr/g" datm.streams
sed -i "s/_yearLast_/$endyr/g" datm.streams
sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm.streams
- # Generate datm_in and nems.configure from model-independent templates:
+ # Generate datm_in and ufs.configure from model-independent templates:
${NCP} ${PARMhafs}/cdeps/datm_in .
sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm_in
- ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./
+ ${NCP} ${PARMforecast}/ufs.configure.cdeps.tmp ./
sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \
-e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \
-e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \
@@ -1247,7 +1251,7 @@ if [ ${run_datm} = yes ]; then
-e "/_system_type_/d" \
-e "s/_atm_model_/datm/g" \
-e "s/_ocn_model_/hycom/g" \
- nems.configure.cdeps.tmp > nems.configure
+ ufs.configure.cdeps.tmp > ufs.configure
elif [ ${run_docn} = yes ]; then
MAKE_MESH_OCN=$( echo "${make_mesh_ocn:-no}" | tr a-z A-Z )
${NLN} "$docn_input_path"/DOCN_input*nc INPUT/
@@ -1272,7 +1276,7 @@ elif [ ${run_docn} = yes ]; then
fi
done
${NLN} "${mesh_ocn}" INPUT/DOCN_ESMF_mesh.nc
- ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./
+ ${NCP} ${PARMforecast}/ufs.configure.cdeps.tmp ./
sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \
-e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \
-e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \
@@ -1286,7 +1290,7 @@ elif [ ${run_docn} = yes ]; then
-e "s/_system_type_/ufs/g" \
-e "s/_atm_model_/fv3/g" \
-e "s/_ocn_model_/docn/g" \
- nems.configure.cdeps.tmp > nems.configure
+ ufs.configure.cdeps.tmp > ufs.configure
fi
# Generate model_configure
@@ -1315,7 +1319,7 @@ OUTPUT_FH=-1
if [ $gtype = regional ]; then
ngrids=${nest_grids}
-elif [ $gtype = nest ]; then
+elif [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then
ngrids=$(( ${nest_grids} + 1 ))
else
echo "FATAL ERROR: Unsupported gtype of ${gtype}. Currently onnly support gtype of nest or regional."
@@ -1434,8 +1438,8 @@ if [ ${write_dopost:-.false.} = .true. ]; then
fi
fi
-# Copy the fd_nems.yaml file
-${NCP} ${HOMEhafs}/sorc/hafs_forecast.fd/tests/parm/fd_nems.yaml ./
+# Copy the fd_ufs.yaml file
+${NCP} ${HOMEhafs}/sorc/hafs_forecast.fd/tests/parm/fd_ufs.yaml ./
# Copy the executable and run the forecast
FORECASTEXEC=${FORECASTEXEC:-${EXEChafs}/hafs_forecast.x}
diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh
index 28bcc5b3b..cb767a2eb 100755
--- a/sorc/build_forecast.sh
+++ b/sorc/build_forecast.sh
@@ -4,19 +4,14 @@ source ./machine-setup.sh > /dev/null 2>&1
if [ $target = wcoss2 ]; then source ../versions/build.ver; fi
cwd=$(pwd)
-cp hafs_forecast.fd/FV3/ccpp/suites_not_used/suite_FV3_HAFS_v1* hafs_forecast.fd/FV3/ccpp/suites/
-cd hafs_forecast.fd/tests
-
-if [ $target = hera ]; then target=hera ; fi
-if [ $target = orion ]; then target=orion ; fi
-if [ $target = jet ]; then target=jet ; fi
-if [ $target = cheyenne ]; then target=cheyenne ; fi
-if [ $target = wcoss2 ]; then target=wcoss2 ; fi
+if [[ $target =~ .*c5 ]] ; then target=gaea-c5 ; fi
+test -s hafs_forecast.fd/FV3/ccpp/suites/suite_FV3_global_nest_v0.xml ||
+ ln -sf ../../../../customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml hafs_forecast.fd/FV3/ccpp/suites/suite_FV3_global_nest_v0.xml
-app=HAFS-MOM6
-./compile.sh "$target" "-DAPP=$app -DREGIONAL_MOM6=ON -DMOVING_NEST=ON -DFASTER=ON -DCCPP_SUITES=FV3_HAFS_v1_thompson_noahmp_nonsst,FV3_HAFS_v1_thompson_noahmp,FV3_HAFS_v1_thompson_nonsst,FV3_HAFS_v1_thompson,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_thompson_tedmf_gfdlsf,FV3_RRFSv2c3 -D32BIT=ON" hafs_mom6 intel YES NO
+cd hafs_forecast.fd/tests
app=HAFSW
-./compile.sh "$target" "-DAPP=$app -DMOVING_NEST=ON -DFASTER=ON -DCCPP_SUITES=FV3_HAFS_v1_thompson_noahmp_nonsst,FV3_HAFS_v1_thompson_noahmp,FV3_HAFS_v1_thompson_nonsst,FV3_HAFS_v1_thompson,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_thompson_tedmf_gfdlsf,FV3_RRFSv2c3 -D32BIT=ON" hafs_hycom intel YES NO
+opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1,FV3_global_nest_v0 -D32BIT=ON"
+./compile.sh "$target" "$opts" 32bit intel YES NO
exit
diff --git a/sorc/build_tools.sh b/sorc/build_tools.sh
index 9325755ba..164fb6628 100755
--- a/sorc/build_tools.sh
+++ b/sorc/build_tools.sh
@@ -21,16 +21,24 @@ elif [ $target = wcoss2 ]; then
export DM_FC="ftn -static"
export DM_F90="ftn -free -static"
export DM_CC="cc -static"
+elif [[ $target =~ gaea ]]; then
+ export FC="ftn"
+ export F90="ftn"
+ export CC=cc
+ export DM_FC="ftn"
+ export DM_F90="ftn"
+ export DM_CC="cc"
else
echo "Unknown machine = $target"
exit 1
fi
-
+export NETCDF="${NETCDF:-$netcdf_c_ROOT}"
+export HDF5="${NETCDF:-$hdf5_ROOT}"
export NETCDF_INCLUDE="-I${NETCDF}/include"
export NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdff -lnetcdf"
export HDF5_INCLUDE=${HDF5_INCLUDE:-"-I${HDF5_INCLUDES:--I${HDF5}/include}"}
export HDF5_LDFLAGS=${HDF5_LDFLAGS:-"-L${HDF5_LIBRARIES:-${HDF5}/lib} -lhdf5_hl -lhdf5"}
-export BUFR_LDFLAGS="${BUFR_LIBd}"
+export BUFR_LDFLAGS="${BUFR_LIB4}"
TOOLS_PATH=${cwd}/hafs_tools.fd
export TOOLS_INC=${TOOLS_PATH}/include
diff --git a/sorc/build_tracker.sh b/sorc/build_tracker.sh
index bb62b3da5..9438fd2ae 100755
--- a/sorc/build_tracker.sh
+++ b/sorc/build_tracker.sh
@@ -4,6 +4,13 @@ source ./machine-setup.sh > /dev/null 2>&1
if [ $target = wcoss2 ]; then source ../versions/build.ver; fi
cwd=$(pwd)
+if [[ ! -s hafs_tracker.fd/modulefiles/gaea-c5.lua ]] ; then
+ pushd hafs_tracker.fd
+ patch -p1 < ../customizations/hafs_tracker.fd/gaea-c5-patch.diff
+ popd
+ cp -fp customizations/hafs_tracker.fd/gaea-c5.lua hafs_tracker.fd/modulefiles/gaea-c5.lua
+fi
+
#export BUILD_TYPE=DEBUG
#export BUILD_TYPE=RELEASE
export BUILD_TYPE=${BUILD_TYPE:-RELEASE}
diff --git a/sorc/build_utils.sh b/sorc/build_utils.sh
index 5260a9032..bc2194b11 100755
--- a/sorc/build_utils.sh
+++ b/sorc/build_utils.sh
@@ -4,6 +4,13 @@ source ./machine-setup.sh > /dev/null 2>&1
if [ $target = wcoss2 ]; then source ../versions/build.ver; fi
cwd=$(pwd)
+if [[ ! -s hafs_utils.fd/modulefiles/build.gaea-c5.intel.lua ]] ; then
+ pushd hafs_utils.fd
+ patch -p1 < ../customizations/hafs_utils.fd/gaea-c5-patch.diff
+ popd
+ cp -fp customizations/hafs_utils.fd/build.gaea-c5.intel.lua hafs_utils.fd/modulefiles/build.gaea-c5.intel.lua
+fi
+
cd hafs_utils.fd
./build_all.sh
diff --git a/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml b/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml
new file mode 100644
index 000000000..36c2c2e41
--- /dev/null
+++ b/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml
@@ -0,0 +1,95 @@
+
+
+
+
+
+
+ GFS_time_vary_pre
+ GFS_rrtmg_setup
+ GFS_rad_time_vary
+ GFS_phys_time_vary
+
+
+
+
+ GFS_suite_interstitial_rad_reset
+ sgscloud_radpre
+ GFS_rrtmg_pre
+ GFS_radiation_surface
+ rad_sw_pre
+ rrtmg_sw
+ rrtmg_sw_post
+ rrtmg_lw_pre
+ rrtmg_lw
+ sgscloud_radpost
+ rrtmg_lw_post
+ GFS_rrtmg_post
+
+
+
+
+ GFS_suite_interstitial_phys_reset
+ GFS_suite_stateout_reset
+ get_prs_fv3
+ GFS_suite_interstitial_1
+ GFS_surface_generic_pre
+ GFS_surface_composites_pre
+ dcyc2t3
+ GFS_surface_composites_inter
+ GFS_suite_interstitial_2
+
+
+
+ mynnsfc_wrapper
+ GFS_surface_loop_control_part1
+ sfc_nst_pre
+ sfc_nst
+ sfc_nst_post
+ noahmpdrv
+ sfc_sice
+ GFS_surface_loop_control_part2
+
+
+
+ GFS_surface_composites_post
+ sfc_diag
+ sfc_diag_post
+ GFS_surface_generic_post
+ mynnedmf_wrapper
+ GFS_GWD_generic_pre
+ unified_ugwp
+ unified_ugwp_post
+ GFS_GWD_generic_post
+ GFS_suite_stateout_update
+ h2ophys
+ get_phi_fv3
+ GFS_suite_interstitial_3
+ GFS_DCNV_generic_pre
+ cu_c3_driver_pre
+ cu_c3_driver
+ GFS_DCNV_generic_post
+ GFS_SCNV_generic_pre
+ GFS_SCNV_generic_post
+ GFS_suite_interstitial_4
+ cnvc90
+ GFS_MP_generic_pre
+ mp_thompson_pre
+
+
+ mp_thompson
+
+
+ mp_thompson_post
+ GFS_MP_generic_post
+ cu_c3_driver_post
+ maximum_hourly_diagnostics
+
+
+
+
+ GFS_stochastics
+ GFS_physics_post
+
+
+
+
diff --git a/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff b/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff
new file mode 100644
index 000000000..aa0e58437
--- /dev/null
+++ b/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff
@@ -0,0 +1,78 @@
+diff --git a/src/build_all_cmake.sh b/src/build_all_cmake.sh
+index 01a7bd7..818d59c 100755
+--- a/src/build_all_cmake.sh
++++ b/src/build_all_cmake.sh
+@@ -1,8 +1,8 @@
+ #!/bin/sh
+
+-set -eux
++# Do not set -eu on GAEA C5 or it'll fail.
+ source ./machine-setup.sh > /dev/null 2>&1
+-
++set -uex
+
+ module use ../modulefiles
+ module load $target
+@@ -24,7 +24,12 @@ elif [ $target = jet ]; then
+ elif [ $target = wcoss2 ] ; then
+ export FC=ftn
+ export F90=ftn
++ cmake_fortran_compiler=ifort
+ export CC=icc
++elif [[ $target =~ gaea ]] ; then
++ export FC=ftn
++ export F90=ftn
++ export CC=cc
+ else
+ echo "Unknown machine = $target"
+ exit 1
+@@ -36,7 +41,7 @@ if [ -d "build" ]; then
+ fi
+ mkdir build
+ cd build
+-cmake .. -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_C_COMPILER=icc -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
++cmake .. -DCMAKE_Fortran_COMPILER=${cmake_fortran_compiler:-$FC} -DCMAKE_C_COMPILER=$CC -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+ make -j 8 VERBOSE=1
+ make install
+
+diff --git a/src/machine-setup.sh b/src/machine-setup.sh
+index 50dee4d..0138b80 100755
+--- a/src/machine-setup.sh
++++ b/src/machine-setup.sh
+@@ -59,18 +59,25 @@ elif [[ -d /glade ]] ; then
+ target=yellowstone
+ module purge
+ elif [[ -d /lustre && -d /ncrc ]] ; then
+- # We are on GAEA.
+- if ( ! eval module help > /dev/null 2>&1 ) ; then
+- # We cannot simply load the module command. The GAEA
+- # /etc/profile modifies a number of module-related variables
+- # before loading the module command. Without those variables,
+- # the module command fails. Hence we actually have to source
+- # /etc/profile here.
+- echo load the module command 1>&2
+- source /etc/profile
++ if [[ $( hostname ) =~ gaea* ]] ; then
++ module purge
++ # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV
++ gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \
++ -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \
++ --pid=$$ --batch
++
++ # Reload system default modules:
++ source /etc/bash.bashrc.local
++
++ # Load EPIC's module directories:
++ source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
++ fi
++
++ if ( head /proc/cpuinfo | grep EPYC > /dev/null ) ; then
++ target=gaea-c5
++ else
++ target=gaea
+ fi
+- target=gaea
+- module purge
+ elif [[ "$(hostname)" =~ "odin" ]]; then
+ target="odin"
+ else
diff --git a/sorc/customizations/hafs_tracker.fd/gaea-c5.lua b/sorc/customizations/hafs_tracker.fd/gaea-c5.lua
new file mode 100644
index 000000000..229e66fd2
--- /dev/null
+++ b/sorc/customizations/hafs_tracker.fd/gaea-c5.lua
@@ -0,0 +1,48 @@
+help([[
+ This module loads libraries required for building and running UPP
+ on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis([===[Loads libraries needed for building the UPP on Gaea ]===])
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+load(pathJoin("stack-python", stack_python_ver))
+
+local ufs_modules = {
+ {["jasper"] = "2.0.32"},
+ {["zlib"] = "1.2.13"},
+ {["libpng"] = "1.6.37"},
+ {["hdf5"] = "1.14.0"},
+ {["netcdf-c"] = "4.9.2"},
+ {["netcdf-fortran"] = "4.6.0"},
+ {["bacio"] = "2.4.1"},
+ {["g2"] = "3.4.5"},
+ {["g2tmpl"] = "1.10.2"},
+ {["w3emc"] = "2.9.2"},
+ {["w3nco"] = "2.4.1"},
+ {["sigio"] = "2.3.2"},
+}
+
+for i = 1, #ufs_modules do
+ for name, default_version in pairs(ufs_modules[i]) do
+ local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+ load(pathJoin(name, os.getenv(env_version_name) or default_version))
+ end
+end
+
+unload("darshan-runtime")
+unload("cray-libsci")
diff --git a/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua b/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua
new file mode 100644
index 000000000..1e32a774f
--- /dev/null
+++ b/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua
@@ -0,0 +1,54 @@
+help([[
+ This module loads libraries required for building and running UPP
+ on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis([===[Loads libraries needed for building the UPP on Gaea ]===])
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+load(pathJoin("stack-python", stack_python_ver))
+
+local ufs_modules = {
+ {["bacio"] = "2.4.1"},
+ {["g2"] = "3.4.5"},
+ {["ip"] = "3.3.3"},
+ {["nemsio"] = "2.5.4"},
+ {["sp"] = "2.3.3"},
+ {["w3nco"] = "2.4.1"},
+ {["sigio"] = "2.3.2"},
+ {["sfcio"] = "1.4.1"},
+ {["zlib"] = "1.2.13"},
+ {["hdf5"] = "1.14.0"},
+ {["netcdf-c"] = "4.9.2"},
+ {["netcdf-fortran"] = "4.6.0"},
+ {["esmf"] = "8.4.2"},
+}
+
+for i = 1, #ufs_modules do
+ for name, default_version in pairs(ufs_modules[i]) do
+ local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+ load(pathJoin(name, os.getenv(env_version_name) or default_version))
+ end
+end
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaea_c5.intel")
diff --git a/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff b/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff
new file mode 100644
index 000000000..71ba8a662
--- /dev/null
+++ b/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff
@@ -0,0 +1,80 @@
+diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
+index 6e73630e..3478959c 100644
+--- a/sorc/machine-setup.sh
++++ b/sorc/machine-setup.sh
+@@ -19,6 +19,14 @@ fi
+ target=""
+ USERNAME=`echo $LOGNAME | awk '{ print tolower($0)'}`
+
++# Disable -e (abort on non-zero exit status) -u (abort on empty or
++# uninitialized variables) and -x (print every command executed)
++# because they can break system scripts.
++__ms_set_e=$( echo $- | grep e && echo YES || echo NO )
++__ms_set_u=$( echo $- | grep u && echo YES || echo NO )
++__ms_set_x=$( echo $- | grep x && echo YES || echo NO )
++set +eux
++
+ if [[ -d /lfs4 ]] ; then
+ # We are on NOAA Jet
+ if ( ! eval module help > /dev/null 2>&1 ) ; then
+@@ -47,43 +55,25 @@ elif [[ -d /glade ]] ; then
+ target=cheyenne
+ module purge
+ elif [[ -d /lustre && -d /ncrc ]] ; then
+- # We are on GAEA.
+- if ( ! eval module help > /dev/null 2>&1 ) ; then
+- # We cannot simply load the module command. The GAEA
+- # /etc/profile modifies a number of module-related variables
+- # before loading the module command. Without those variables,
+- # the module command fails. Hence we actually have to source
+- # /etc/profile here.
+- source /etc/profile
+- __ms_source_etc_profile=yes
++ if [[ $( hostname ) =~ gaea* ]] ; then
++ # Only runs on head nodes
++ module purge
++ # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV
++ gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \
++ -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \
++ --pid=$$ --batch
++
++ # Reload system default modules:
++ source /etc/bash.bashrc.local
++
++ # Load EPIC's module directories:
++ source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
++ fi
++ if ( head /proc/cpuinfo | grep EPYC > /dev/null ) ; then
++ target=gaea-c5
+ else
+- __ms_source_etc_profile=no
+- fi
+- module purge > /dev/null 2>&1
+- module purge
+-# clean up after purge
+- unset _LMFILES_
+- unset _LMFILES_000
+- unset _LMFILES_001
+- unset LOADEDMODULES
+- module load modules
+- if [[ -d /opt/cray/ari/modulefiles ]] ; then
+- module use -a /opt/cray/ari/modulefiles
+- fi
+- if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then
+- module use -a /opt/cray/pe/ari/modulefiles
+- fi
+- if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then
+- module use -a /opt/cray/pe/craype/default/modulefiles
+- fi
+- if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then
+- source /etc/opt/cray/pe/admin-pe/site-config
+- fi
+- if [[ "$__ms_source_etc_profile" == yes ]] ; then
+- source /etc/profile
+- unset __ms_source_etc_profile
++ target=gaea
+ fi
+- target=gaea
+ elif [[ "$(hostname)" =~ "Orion" ]]; then
+ target="orion"
+ module purge
diff --git a/sorc/hafs_build.cfg b/sorc/hafs_build.cfg
index c7314ac7d..ba9c97475 100644
--- a/sorc/hafs_build.cfg
+++ b/sorc/hafs_build.cfg
@@ -7,9 +7,9 @@
Building tracker (tracker) ............................ yes
Building utils (utils) ................................ yes
Building tools (tools) ................................ yes
- Building gsi (gsi) .................................... yes
- Building hycom_utils (hycom_utils) .................... yes
- Building ww3_utils (ww3_utils) ........................ yes
+ Building gsi (gsi) .................................... no
+ Building hycom_utils (hycom_utils) .................... no
+ Building ww3_utils (ww3_utils) ........................ no
# -- END --
diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd
index d972f5a8a..f42e347b1 160000
--- a/sorc/hafs_forecast.fd
+++ b/sorc/hafs_forecast.fd
@@ -1 +1 @@
-Subproject commit d972f5a8a13d075f09294c59f156844c63e3df57
+Subproject commit f42e347b1cad15b4b2470fecb17cb19bb2d19459
diff --git a/sorc/hafs_graphics.fd/pygraf b/sorc/hafs_graphics.fd/pygraf
index 84e411224..9003a692b 160000
--- a/sorc/hafs_graphics.fd/pygraf
+++ b/sorc/hafs_graphics.fd/pygraf
@@ -1 +1 @@
-Subproject commit 84e4112245e10df1d95b7f2dd6426673b71f913f
+Subproject commit 9003a692bc94b3561f78844585ad9165b2d6bfea
diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd
index 67258a934..8c86e8d41 160000
--- a/sorc/hafs_gsi.fd
+++ b/sorc/hafs_gsi.fd
@@ -1 +1 @@
-Subproject commit 67258a934ebf8e0e2d0cf2c914796f853b6f73e6
+Subproject commit 8c86e8d418ad91ed25da7485f63704754ebaf77f
diff --git a/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt b/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt
index 7f8625d78..9aa5684c1 100644
--- a/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt
+++ b/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt
@@ -87,7 +87,7 @@ target_link_options(${exe_name} PRIVATE
target_link_libraries(
${exe_name}
- bufr::bufr_d
+ bufr::bufr_4
NetCDF::NetCDF_Fortran
NetCDF::NetCDF_C
MPI::MPI_Fortran)
diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd
index 21274b39c..97a3b75dd 160000
--- a/sorc/hafs_tracker.fd
+++ b/sorc/hafs_tracker.fd
@@ -1 +1 @@
-Subproject commit 21274b39ca6c59322a1d2f6d5e67d98261f2865b
+Subproject commit 97a3b75dd40aef1b39b3cf436a8b2944082a87ae
diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd
index 9d1622b35..340d87cec 160000
--- a/sorc/hafs_utils.fd
+++ b/sorc/hafs_utils.fd
@@ -1 +1 @@
-Subproject commit 9d1622b3552d762160a20ce79969da2076a84513
+Subproject commit 340d87cecd245ae2e97410e85a0860573e04a014
diff --git a/sorc/install_all.sh b/sorc/install_all.sh
index 1f16376f3..9132de603 100755
--- a/sorc/install_all.sh
+++ b/sorc/install_all.sh
@@ -15,8 +15,9 @@ CP='/bin/cp -rp'
# install forecast
#------------------------------------
$Build_forecast && {
- ${CP} hafs_forecast.fd/tests/fv3_hafs_mom6.exe ../exec/hafs_forecast_mom6.x
- ${CP} hafs_forecast.fd/tests/fv3_hafs_hycom.exe ../exec/hafs_forecast_hycom.x
+ ${CP} hafs_forecast.fd/tests/fv3_32bit.exe ../exec/hafs_forecast_hycom.x
+# ${CP} hafs_forecast.fd/tests/fv3_hafs_mom6.exe ../exec/hafs_forecast_mom6.x
+# ${CP} hafs_forecast.fd/tests/fv3_hafs_hycom.exe ../exec/hafs_forecast_hycom.x
}
#------------------------------------
diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh
index b995aea25..15c7810b5 100755
--- a/sorc/link_fix.sh
+++ b/sorc/link_fix.sh
@@ -21,6 +21,8 @@ elif [ ${target} == "orion" ]; then
FIXROOT=/work/noaa/hwrf/noscrub/hafs-fix-files/hafs-${FIXversion}-fix/fix
elif [ ${target} == "jet" ]; then
FIXROOT=/lfs4/HFIP/hwrf-data/hafs-fix-files/hafs-${FIXversion}-fix/fix
+elif [[ $target =~ gaea ]]; then
+ FIXROOT=/lustre/f2/dev/Samuel.Trahan/hafs/hafs-fix-files/hafs-${FIXversion}-fix/fix
else
echo "FATAL ERROR: Unknown site " ${target}
exit 1
diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py
index 14438bc25..ea58f828d 100644
--- a/ush/hafs/launcher.py
+++ b/ush/hafs/launcher.py
@@ -578,6 +578,10 @@ def launch(file_list,cycle,stid,moreopt,case_root,init_dirs=True,
revit=conf.read_fake_tcvitals()
conf.gen_vitals(stid,cycling_interval,revit)
+ if conf.getbool('config', 'run_pygraf') and not conf.getbool('config','pygraf_can_run'):
+ logger.warning("Will disable pygraf because it cannot run on this system (config.pygraf_can_run=no).")
+ conf.set('config', 'run_pygraf', 'no')
+
# rocoto does not initialize the dirs, it returns here.
if not init_dirs:
if prelaunch is not None:
diff --git a/ush/hafs_pre_job.sh.inc b/ush/hafs_pre_job.sh.inc
index f54c8c09d..aceaae3f8 100644
--- a/ush/hafs_pre_job.sh.inc
+++ b/ush/hafs_pre_job.sh.inc
@@ -73,18 +73,14 @@ elif [[ -d /glade ]] ; then
target=cheyenne
module purge
elif [[ -d /lustre && -d /ncrc ]] ; then
- # We are on GAEA.
- if ( ! eval module help > /dev/null 2>&1 ) ; then
- # We cannot simply load the module command. The GAEA
- # /etc/profile modifies a number of module-related variables
- # before loading the module command. Without those variables,
- # the module command fails. Hence we actually have to source
- # /etc/profile here.
- echo load the module command 1>&2
- source /etc/profile
+ # You cannot "module purge" on GAEA
+ if ( head /proc/cpuinfo | grep AuthenticAMD > /dev/null ) ; then
+ target=gaea_c5
+ else
+ target=gaea
fi
- target=gaea
- module purge
+ ulimit -s unlimited
+ export TZ=UTC
elif [[ "$(hostname)" =~ "odin" ]] ; then
target="odin"
else
diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc
index c50260b9b..b4997d4a1 100644
--- a/ush/hafs_runcmd.sh.inc
+++ b/ush/hafs_runcmd.sh.inc
@@ -11,8 +11,8 @@ export PURE_OMP_THREADS=${PURE_OMP_THREADS:-${OMP_NUM_THREADS:-24}}
export NCNODE=${NCNODE:-$(($NCTSK * $OMP_NUM_THREADS))}
export KMP_AFFINITY=disabled
export KMP_STACKSIZE=${KMP_STACKSIZE:-2048m}
-#export OMP_STACKSIZE=${OMP_STACKSIZE:-2048m}
-# Avoild "NetCDF: HDF error"
+export OMP_STACKSIZE=${OMP_STACKSIZE:-2048m}
+# Avoid "NetCDF: HDF error"
export HDF5_USE_FILE_LOCKING=FALSE
# Set job run commands
@@ -41,6 +41,15 @@ elif [ "$machine" = jet ] || [ "$machine" = hera ] || [ "$machine" = orion ]; th
export APRUNF="time"
export APRUNCFP="time"
export BACKGROUND="&"
+elif [ "$machine" = gaea_c5 ] ; then
+ export APRUN="time"
+ export APRUNS="srun -l --mem=0 --exclusive --ntasks=1 --ntasks-per-node=1 --cpus-per-task=128 "
+ export APRUNO="srun -l --mem=0 --exclusive --ntasks=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS} -- "
+ export APRUNC="srun -l --mem=0 --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS} -- "
+ export APRUNM="$APRUNC"
+ export APRUNF="$APRUNC ${MPISERIAL:-mpiserial} -m "
+ export APRUNCFP="$APRUN"
+ export BACKGROUND=""
else
echo "WARNING: unknown platform. Guessing the job run commands..."
export APRUN="time"
diff --git a/ush/produtil/cluster.py b/ush/produtil/cluster.py
index b8ac04e6f..dac610d7a 100644
--- a/ush/produtil/cluster.py
+++ b/ush/produtil/cluster.py
@@ -99,7 +99,12 @@ def where():
elif os.path.exists('/lfs/h2/emc'):
here=WCOSS2()
elif os.path.exists('/lustre/f2'):
- here=NOAAGAEA()
+ with open("/proc/cpuinfo", "rt") as fd:
+ is_amd=fd.read(3000).find('EPYC') >= 0
+ if is_amd:
+ here=NOAAGAEAC5()
+ else:
+ here=NOAAGAEA()
else:
here=Cluster(False,False,False,'noname','noname')
return here
@@ -178,6 +183,14 @@ def __init__(self):
super(NOAAGAEA,self).__init__(False,True,False,'gaea',
'gaea.rdhpcs.noaa.gov')
+class NOAAGAEAC5(Cluster):
+ """!Represents the NOAA GAEA cluster C5 partition. Allows ACLs to be used for
+ restricted data, and specifies that group quotas are not in use."""
+ def __init__(self):
+ """!constructor for NOAAGAEAC5"""
+ super(NOAAGAEAC5,self).__init__(False,True,False,'gaea_c5',
+ 'gaea.rdhpcs.noaa.gov')
+
class NOAAHera(Cluster):
"""!Represents the NOAA Hera cluster. Does not allow ACLs,
assumes no group quotas (fileset quotas instead)."""