From b950540b51cc6253098d7c7b55b3c7329740da00 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 21 Nov 2023 18:24:30 -0500 Subject: [PATCH 01/30] gaea c5 port --- .gitmodules | 14 +-- modulefiles/hafs.gaea_c5.lua | 79 +++++++++++++++++ parm/WaterInTheWest.conf | 16 ++-- parm/WaterInTheWestC192.conf | 9 +- parm/WaterInTheWestC384.conf | 12 +-- parm/WaterInTheWestC96.conf | 14 +-- ...onfigure.atmonly => ufs.configure.atmonly} | 0 ...onfigure.atmonly => ufs.configure.atmonly} | 0 ...gure.cdeps.tmp => ufs.configure.cdeps.tmp} | 0 ...onfigure.cpl.tmp => ufs.configure.cpl.tmp} | 0 ...figure.mom6.tmp => ufs.configure.mom6.tmp} | 0 parm/system.conf.gaea | 51 +++++++++++ rocoto/WaterInTheWest.sh | 2 +- rocoto/sites/all.ent | 1 + rocoto/sites/gaea_c5.ent | 85 +++++++++++++++++++ rocoto/sites/hera.ent | 1 + rocoto/sites/kjet.ent | 1 + rocoto/sites/orion.ent | 1 + rocoto/sites/wcoss2.ent | 1 + rocoto/sites/xjet.ent | 1 + scripts/exhafs_atm_post.sh | 10 +-- scripts/exhafs_forecast.sh | 30 +++---- sorc/build_forecast.sh | 15 +--- sorc/build_tools.sh | 10 ++- sorc/hafs_gsi.fd | 2 +- sorc/hafs_tracker.fd | 2 +- sorc/hafs_utils.fd | 2 +- sorc/install_all.sh | 5 +- sorc/link_fix.sh | 2 + ush/hafs_pre_job.sh.inc | 18 ++-- ush/hafs_runcmd.sh.inc | 13 ++- ush/produtil/cluster.py | 15 +++- 32 files changed, 330 insertions(+), 82 deletions(-) create mode 100644 modulefiles/hafs.gaea_c5.lua rename parm/forecast/globnest/{nems.configure.atmonly => ufs.configure.atmonly} (100%) rename parm/forecast/regional/{nems.configure.atmonly => ufs.configure.atmonly} (100%) rename parm/forecast/regional/{nems.configure.cdeps.tmp => ufs.configure.cdeps.tmp} (100%) rename parm/forecast/regional/{nems.configure.cpl.tmp => ufs.configure.cpl.tmp} (100%) rename parm/forecast/regional/{nems.configure.mom6.tmp => ufs.configure.mom6.tmp} (100%) create mode 100644 parm/system.conf.gaea create mode 100644 rocoto/sites/gaea_c5.ent diff --git a/.gitmodules b/.gitmodules index 80130d396..80bb1eea7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd - url = https://github.com/NOAA-GSL/ufs-weather-model.git - branch = WaterInTheWest + url = https://github.com/SamuelTrahanNOAA/ufs-weather-model.git + branch = WaterInTheWestC5-Nov2023 # url = https://github.com/hafs-community/ufs-weather-model.git # branch = support/hafs.v1.1.0 # url = https://github.com/ufs-community/ufs-weather-model.git @@ -10,18 +10,18 @@ path = sorc/hafs_utils.fd # url = https://github.com/hafs-community/UFS_UTILS.git # branch = support/HAFS - url = https://github.com/ufs-community/UFS_UTILS.git - branch = develop + url = https://github.com/SamuelTrahanNOAA/UFS_UTILS.git + branch = WaterInTheWestC5-Nov2023 [submodule "gfdl-tracker"] path = sorc/hafs_tracker.fd url = https://github.com/SamuelTrahanNOAA/gfdl-tracker.git # branch = support/HAFS - branch = support/hafs.v1.1.0 + branch = WaterInTheWestC5-Nov2023 [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd - url = https://github.com/hafs-community/GSI.git + url = https://github.com/SamuelTrahanNOAA/GSI.git # branch = support/HAFS - branch = support/hafs.v1.1.0 + branch = WaterInTheWestC5-Nov2023 [submodule "GPLOT"] path = sorc/hafs_graphics.fd/hrd_gplot url = https://github.com/hafs-community/GPLOT.git diff --git a/modulefiles/hafs.gaea_c5.lua b/modulefiles/hafs.gaea_c5.lua new file mode 100644 index 000000000..b7cebcf58 --- /dev/null +++ b/modulefiles/hafs.gaea_c5.lua @@ -0,0 +1,79 @@ +help([[ + This module loads libraries required for building and running HAFS + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. +]]) + +whatis("Description: HAFS Application environment") + +load("PrgEnv-intel/8.3.3") +load("intel-classic/2023.1.0") +load("cray-mpich/8.1.25") +-- load("python/3.9.12") + +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles") +prepend_path("MODULEPATH", "/lustre/f2/dev/Samuel.Trahan/hafs/modulefiles/") + +miniconda_ver=os.getenv("miniconda_ver") or "23.5.2-hafs" +load(pathJoin("miniconda", miniconda_ver)) + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +-- stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +-- load(pathJoin("stack-python", stack_python_ver)) + +local ufs_modules = { + {["jasper"] = "2.0.32"}, + {["zlib"] = "1.2.13"}, + {["libpng"] = "1.6.37"}, + {["hdf5"] = "1.14.0"}, + {["netcdf-c"] = "4.9.2"}, + {["netcdf-fortran"] = "4.6.0"}, + {["parallelio"] = "2.5.10"}, + {["esmf"] = "8.4.2"}, + {["fms"] = "2023.01"}, + {["bacio"] = "2.4.1"}, + {["crtm"] = "2.4.0"}, + {["g2"] = "3.4.5"}, + {["g2tmpl"] = "1.10.2"}, + {["ip"] = "3.3.3"}, + {["nemsio"] = "2.5.4"}, + {["sp"] = "2.3.3"}, + {["w3emc"] = "2.9.2"}, + {["w3nco"] = "2.4.1"}, + {["gftl-shared"] = "1.5.0"}, +-- no yafyaml + {["mapl"] = "2.35.2-esmf-8.4.2"}, + {["bufr"] = "11.7.0"}, + {["sfcio"] = "1.4.1"}, + {["sigio"] = "2.3.2"}, +-- no szip + {["wrf-io"] = "1.2.0"}, + {["prod-util"] = "1.2.2"}, + {["grib-util"] = "1.2.3"}, + {["wgrib2"] = "2.0.8"}, +-- no gempak + {["nco"] = "5.0.6"}, + {["cdo"] = "2.0.5"}, +} + +for i = 1, #ufs_modules do + for name, default_version in pairs(ufs_modules[i]) do + local env_version_name = string.gsub(name, "-", "_") .. "_ver" + load(pathJoin(name, os.getenv(env_version_name) or default_version)) + end +end + +load("rocoto") + +unload("darshan-runtime") +unload("cray-libsci") + +setenv("CMAKE_C_COMPILER", "cc") +setenv("CMAKE_CXX_COMPILER", "CC") +setenv("CMAKE_Fortran_COMPILER", "ftn") +setenv("CMAKE_Platform", "gaea_c5.intel") diff --git a/parm/WaterInTheWest.conf b/parm/WaterInTheWest.conf index e0adac941..1f1968277 100644 --- a/parm/WaterInTheWest.conf +++ b/parm/WaterInTheWest.conf @@ -69,9 +69,9 @@ jend_nest=1294 regional_esg=no [forecast] -ccpp_suite_regional=FV3_RRFSv2c3 -ccpp_suite_glob=FV3_RRFSv2c3 -ccpp_suite_nest=FV3_RRFSv2c3 +ccpp_suite_regional=FV3_global_nest_v1 +ccpp_suite_glob=FV3_global_nest_v1 +ccpp_suite_nest=FV3_global_nest_v1 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 @@ -111,7 +111,8 @@ npz=64 is_moving_nest=.false. quilting=.true. -write_dopost=.false. +write_dopost=.true. +write_tasks_per_group=360 output_history=.true. output_grid=regional_latlon,regional_latlon @@ -160,8 +161,8 @@ lightning_threat=.true. satpost=.false. post_gridspecs=auto,auto trak_gridspecs={post_gridspecs} -postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt -postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt +# postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt +# postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt [atm_post_ens] satpost_ens=.false. @@ -169,4 +170,5 @@ post_gridspecs_ens=auto,auto trak_gridspecs_ens={post_gridspecs} [rocotostr] -FORECAST_RESOURCES=FORECAST_RESOURCES_2460PE +FORECAST_RESOURCES=FORECAST_RESOURCES_3060PE +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES \ No newline at end of file diff --git a/parm/WaterInTheWestC192.conf b/parm/WaterInTheWestC192.conf index 2fa2f02b5..4a43917bf 100644 --- a/parm/WaterInTheWestC192.conf +++ b/parm/WaterInTheWestC192.conf @@ -69,9 +69,9 @@ jend_nest=322 regional_esg=no [forecast] -ccpp_suite_regional=FV3_RRFSv2c3 -ccpp_suite_glob=FV3_RRFSv2c3 -ccpp_suite_nest=FV3_RRFSv2c3 +ccpp_suite_regional=FV3_global_nest_v1 +ccpp_suite_glob=FV3_global_nest_v1 +ccpp_suite_nest=FV3_global_nest_v1 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 @@ -113,7 +113,7 @@ npz=64 is_moving_nest=.false. quilting=.true. -write_dopost=.false. +write_dopost=.true. output_history=.true. output_grid=regional_latlon,regional_latlon @@ -174,3 +174,4 @@ trak_gridspecs_ens={post_gridspecs} [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_600PE +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES diff --git a/parm/WaterInTheWestC384.conf b/parm/WaterInTheWestC384.conf index de7feb798..e1fc616d4 100644 --- a/parm/WaterInTheWestC384.conf +++ b/parm/WaterInTheWestC384.conf @@ -69,9 +69,9 @@ jend_nest=648 regional_esg=no [forecast] -ccpp_suite_regional=FV3_RRFSv2c3 -ccpp_suite_glob=FV3_RRFSv2c3 -ccpp_suite_nest=FV3_RRFSv2c3 +ccpp_suite_regional=FV3_global_nest_v1 +ccpp_suite_glob=FV3_global_nest_v1 +ccpp_suite_nest=FV3_global_nest_v1 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 @@ -111,7 +111,8 @@ npz=64 is_moving_nest=.false. quilting=.true. -write_dopost=.false. +write_dopost=.true. +write_tasks_per_group=120 output_history=.true. output_grid=regional_latlon,regional_latlon @@ -171,4 +172,5 @@ post_gridspecs_ens=auto,auto trak_gridspecs_ens={post_gridspecs} [rocotostr] -FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE +FORECAST_RESOURCES=FORECAST_RESOURCES_1440PE +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES diff --git a/parm/WaterInTheWestC96.conf b/parm/WaterInTheWestC96.conf index cf5a1b6bc..8d37d9e2d 100644 --- a/parm/WaterInTheWestC96.conf +++ b/parm/WaterInTheWestC96.conf @@ -69,9 +69,9 @@ jend_nest=160 regional_esg=no [forecast] -ccpp_suite_regional=FV3_RRFSv2c3 -ccpp_suite_glob=FV3_RRFSv2c3 -ccpp_suite_nest=FV3_RRFSv2c3 +ccpp_suite_regional=FV3_global_nest_v1 +ccpp_suite_glob=FV3_global_nest_v1 +ccpp_suite_nest=FV3_global_nest_v1 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 @@ -113,7 +113,7 @@ npz=64 is_moving_nest=.false. quilting=.true. -write_dopost=.false. +write_dopost=.true. output_history=.true. output_grid=regional_latlon,regional_latlon @@ -163,8 +163,8 @@ lightning_threat=.true. satpost=.false. post_gridspecs=auto,auto trak_gridspecs={post_gridspecs} -postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt -postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt +# postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt +# postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt [atm_post_ens] satpost_ens=.false. @@ -173,4 +173,4 @@ trak_gridspecs_ens={post_gridspecs} [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_240PE -ATM_POST_RESOURCES=ATM_POST_SMALL_RESOURCES +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES \ No newline at end of file diff --git a/parm/forecast/globnest/nems.configure.atmonly b/parm/forecast/globnest/ufs.configure.atmonly similarity index 100% rename from parm/forecast/globnest/nems.configure.atmonly rename to parm/forecast/globnest/ufs.configure.atmonly diff --git a/parm/forecast/regional/nems.configure.atmonly b/parm/forecast/regional/ufs.configure.atmonly similarity index 100% rename from parm/forecast/regional/nems.configure.atmonly rename to parm/forecast/regional/ufs.configure.atmonly diff --git a/parm/forecast/regional/nems.configure.cdeps.tmp b/parm/forecast/regional/ufs.configure.cdeps.tmp similarity index 100% rename from parm/forecast/regional/nems.configure.cdeps.tmp rename to parm/forecast/regional/ufs.configure.cdeps.tmp diff --git a/parm/forecast/regional/nems.configure.cpl.tmp b/parm/forecast/regional/ufs.configure.cpl.tmp similarity index 100% rename from parm/forecast/regional/nems.configure.cpl.tmp rename to parm/forecast/regional/ufs.configure.cpl.tmp diff --git a/parm/forecast/regional/nems.configure.mom6.tmp b/parm/forecast/regional/ufs.configure.mom6.tmp similarity index 100% rename from parm/forecast/regional/nems.configure.mom6.tmp rename to parm/forecast/regional/ufs.configure.mom6.tmp diff --git a/parm/system.conf.gaea b/parm/system.conf.gaea new file mode 100644 index 000000000..f3592c912 --- /dev/null +++ b/parm/system.conf.gaea @@ -0,0 +1,51 @@ +## This is the system-specific configuration file for Hera +[config] +## Project disk area +disk_project=esrl +## Project hpss tape area +tape_project=wrfruc +## CPU account name for submitting jobs to the batch system. +cpu_account=gsl-glo +## Archive path +archive=hpss:/BMC/{tape_project}/2year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar + +[dir] +gaea_input=/lustre/f2/dev/Samuel.Trahan/hafs/hafs-input +hafs_work=/lustre/f2/dev/{ENV[USER]}/hafs/work + +## Save directory. Make sure you edit this. +CDSAVE={hafs_work} +## Non-scrubbed directory for track files, etc. Make sure you edit this. +CDNOSCRUB={hafs_work}/noscrub/hafstrak +## Scrubbed directory for large work files. Make sure you edit this. +CDSCRUB=/lustre/f2/scratch/{ENV[USER]} +## Syndat directory for finding which cycles to run +syndat={gaea_input}/SYNDAT-PLUS +#oldcom={ENV[COMOLD]} +COMOLD={oldcom} +COMIN={COMhafs} +COMOUT={COMhafs} +COMINnhc={ENV[DCOMROOT|-/dcom]}/nhc/atcf/ncep +COMINjtwc={ENV[DCOMROOT|-/dcom]}/{ENV[PDY]}/wtxtbul/storm_data +COMgfs={gaea_input}/COMGFSv16 +COMINobs={COMgfs} +COMINgfs={COMgfs} +COMINgdas={COMgfs} +COMINarch={COMgfs}/syndat +COMrtofs={gaea_input}/COMRTOFSv2 +COMINrtofs={COMrtofs} +COMINmsg={COMINgfs} +COMINhafs={COMINgfs} +DATMdir={hafs_work}/noscrub/DATM +DOCNdir={hafs_work}/noscrub/DOCN +## A-Deck directory for graphics +ADECKhafs={gaea_input}/aid +## B-Deck directory for graphics +BDECKhafs={gaea_input}/btk +## cartopyDataDir directory for graphics +cartopyDataDir={gaea_input}/cartopy + +[holdvars] +WHERE_AM_I=gaea_c5 +WHICH_JET=none + diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh index f28182fd7..44dec0b7b 100755 --- a/rocoto/WaterInTheWest.sh +++ b/rocoto/WaterInTheWest.sh @@ -7,7 +7,7 @@ set -x cd ${HOMEhafs}/rocoto EXPT=$(basename ${HOMEhafs}) -SUBEXPT=$EXPT +SUBEXPT=C768 conf=../parm/WaterInTheWest.conf opts="-t -f" scrubopt="config.scrub_work=no config.scrub_com=no" diff --git a/rocoto/sites/all.ent b/rocoto/sites/all.ent index e2e3b6680..cafba9c6a 100644 --- a/rocoto/sites/all.ent +++ b/rocoto/sites/all.ent @@ -9,3 +9,4 @@ + diff --git a/rocoto/sites/gaea_c5.ent b/rocoto/sites/gaea_c5.ent new file mode 100644 index 000000000..1724aee90 --- /dev/null +++ b/rocoto/sites/gaea_c5.ent @@ -0,0 +1,85 @@ + + + + + + + + + + + + --clusters=c5--exclusive"> + + + + 128&C5_EXTRA;"> + 128&C5_EXTRA;"> + + + + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:00"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:00"> + 1:ppn=128TOTAL_TASKS18NCTSK6OMP_THREADS601:30:00"> + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS100:15:00"> + 3:ppn=60:tpp=1TOTAL_TASKS180NCTSK60OMP_THREADS100:15:00"> + 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS101:30:00"> + 3:ppn=60:tpp=1TOTAL_TASKS180NCTSK60OMP_THREADS100:30:00"> + 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS101:30:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> + 1:ppn=8:tpp=16TOTAL_TASKS8NCTSK8OMP_THREADS1600:30:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> + 10:ppn=16:tpp=8TOTAL_TASKS160NCTSK16OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00"> + 21:ppn=1:tpp=60TOTAL_TASKS21NCTSK1OMP_THREADS60-l place=vscatter:excl02:00:00"> + 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS1-l place=vscatter:excl02:00:00"> + 1:ppn=8:tpp=1TOTAL_TASKS8NCTSK8OMP_THREADS100:10:00"> + 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00"> + + 02:59:00"> + OMP_THREADS2"> + + + 4:ppn=60:tpp=2TOTAL_TASKS240NCTSK60&FORECAST_EXTRA;"> + 6:ppn=60:tpp=2TOTAL_TASKS360NCTSK60&FORECAST_EXTRA;"> + 9:ppn=60:tpp=2TOTAL_TASKS540NCTSK60&FORECAST_EXTRA;"> + 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60&FORECAST_EXTRA;"> + 11:ppn=60:tpp=2TOTAL_TASKS660NCTSK60&FORECAST_EXTRA;"> + 12:ppn=60:tpp=2TOTAL_TASKS720NCTSK60&FORECAST_EXTRA;"> + 13:ppn=60:tpp=2TOTAL_TASKS780NCTSK60&FORECAST_EXTRA;"> + 14:ppn=60:tpp=2TOTAL_TASKS840NCTSK60&FORECAST_EXTRA;"> + 15:ppn=60:tpp=2TOTAL_TASKS900NCTSK60&FORECAST_EXTRA;"> + 16:ppn=60:tpp=2TOTAL_TASKS960NCTSK60&FORECAST_EXTRA;"> + 17:ppn=60:tpp=2TOTAL_TASKS1020NCTSK60&FORECAST_EXTRA;"> + 18:ppn=60:tpp=2TOTAL_TASKS1080NCTSK60&FORECAST_EXTRA;"> + 19:ppn=60:tpp=2TOTAL_TASKS1140NCTSK60&FORECAST_EXTRA;"> + 20:ppn=60:tpp=2TOTAL_TASKS1200NCTSK60&FORECAST_EXTRA;"> + 21:ppn=60:tpp=2TOTAL_TASKS1260NCTSK60&FORECAST_EXTRA;"> + 22:ppn=60:tpp=2TOTAL_TASKS1320NCTSK60&FORECAST_EXTRA;"> + 23:ppn=60:tpp=2TOTAL_TASKS1380NCTSK60&FORECAST_EXTRA;"> + 24:ppn=60:tpp=2TOTAL_TASKS1440NCTSK60&FORECAST_EXTRA;"> + 30:ppn=60:tpp=2TOTAL_TASKS1800NCTSK60&FORECAST_EXTRA;"> + 36:ppn=60:tpp=2TOTAL_TASKS2160NCTSK60&FORECAST_EXTRA;"> + 41:ppn=60:tpp=2TOTAL_TASKS2460NCTSK60&FORECAST_EXTRA;"> + 43:ppn=60:tpp=2TOTAL_TASKS2580NCTSK60&FORECAST_EXTRA;"> + 45:ppn=60:tpp=2TOTAL_TASKS2700NCTSK60&FORECAST_EXTRA;"> + 46:ppn=60:tpp=2TOTAL_TASKS2760NCTSK60&FORECAST_EXTRA;"> + 47:ppn=60:tpp=2TOTAL_TASKS2820NCTSK60&FORECAST_EXTRA;"> + 49:ppn=60:tpp=2TOTAL_TASKS2940NCTSK60&FORECAST_EXTRA;"> + 51:ppn=60:tpp=2TOTAL_TASKS3060NCTSK60&FORECAST_EXTRA;"> + + 90:ppn=20:tpp=2TOTAL_TASKS1800NCTSK20&FORECAST_EXTRA;"> + + 1:ppn=40:tpp=3TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> + 1:ppn=40:tpp=3TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> + 10:ppn=40:tpp=3TOTAL_TASKS400NCTSK40OMP_THREADS107:59:00"> + 10:ppn=40:tpp=3TOTAL_TASKS400NCTSK40OMP_THREADS107:59:00"> + + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS100:30:00"> + 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00"> + 1:ppn=1:tpp=1TOTAL_TASKS2NCTSK1OMP_THREADS100:15:00"> + 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00"> + + 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS107:59:00"> + 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS107:59:00"> diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index eb7a20b51..8347dadbf 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -64,6 +64,7 @@ 150:ppn=20:tpp=2TOTAL_TASKS3000NCTSK20&FORECAST_EXTRA;"> 153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;"> + 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00"> 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00"> 6:ppn=20:tpp=1TOTAL_TASKS120NCTSK20OMP_THREADS107:59:00"> 8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20OMP_THREADS207:59:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index c31df72a8..9d834d0fc 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -60,6 +60,7 @@ 147:ppn=20:tpp=2TOTAL_TASKS2940NCTSK20&FORECAST_EXTRA;"> 153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> 3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00"> 3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00"> 3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 25400d95e..f09702e9b 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -63,6 +63,7 @@ 150:ppn=20:tpp=2TOTAL_TASKS3000NCTSK20&FORECAST_EXTRA;"> 153:ppn=20:tpp=2TOTAL_TASKS3060NCTSK20&FORECAST_EXTRA;"> + 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00"> 1:ppn=20:tpp=2TOTAL_TASKS20NCTSK20OMP_THREADS207:59:00"> 3:ppn=40:tpp=1TOTAL_TASKS120NCTSK40OMP_THREADS107:59:00"> 8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20OMP_THREADS207:59:00"> diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index c1ac73c9d..cc5854875 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -61,6 +61,7 @@ 49:ppn=60:tpp=2TOTAL_TASKS2940NCTSK60&FORECAST_EXTRA;"> 51:ppn=60:tpp=2TOTAL_TASKS3060NCTSK60&FORECAST_EXTRA;"> + 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00"> 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00"> 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00"> 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS1-l place=vscatter:excl07:59:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 545eb86a8..1f4ab1c6e 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -60,6 +60,7 @@ 245:ppn=12:tpp=2TOTAL_TASKS2940NCTSK12&FORECAST_EXTRA;"> 255:ppn=12:tpp=2TOTAL_TASKS3060NCTSK12&FORECAST_EXTRA;"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> 5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00"> 5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00"> 5:ppn=24:tpp=1TOTAL_TASKS120NCTSK24OMP_THREADS107:59:00"> diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index 47989ab11..7594d8809 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -191,13 +191,13 @@ if [ ${write_dopost:-.false.} = .true. ]; then # Wait for model output n=1 -hurf=${INPdir}/HURPRS${neststr}.GrbF${FHR2} +hurf=${INPdir}/HURPRS.GrbF${FHR2}${neststr} while [ $n -le 360 ]; do - if [ ! -s ${INPdir}/log.atm.f${FHR3} ] || [ ! -s ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ]; then + if [ ! -s ${INPdir}/log.atm.f${FHR3} ] || [ ! -s ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ]; then echo "${INPdir}/log.atm.f${FHR3} not ready, sleep 10s" sleep 10s else - echo "${INPdir}/log.atm.f${FHR3}, ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ready, continue" + echo "${INPdir}/log.atm.f${FHR3}, ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ready, continue" sleep 1s break fi @@ -243,9 +243,9 @@ cd ${DATA_POST} # Note: Currently the inline post (write_dopost) does not support nesting configurations yet. if [ ${write_dopost:-.false.} = .true. ]; then -${NCP} -p ${INPdir}/HURPRS${neststr}.GrbF${FHR2} ${grb2post} +${NCP} -p ${INPdir}/HURPRS.GrbF${FHR2}${neststr} ${grb2post} if [ ${satpost} = .true. ]; then - ${NCP} -p ${INPdir}/HURSAT${neststr}.GrbF${FHR2} ${sat_grb2post} + ${NCP} -p ${INPdir}/HURSAT.GrbF${FHR2}${neststr} ${sat_grb2post} fi else diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index a673f03c3..a39ad9864 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -736,7 +736,7 @@ fi cd .. -# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and nems.configure +# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and ufs.configure ${NCP} ${PARMforecast}/diag_table.tmp . if [ ${imp_physics:-11} = 8 ]; then ${NCP} ${PARMforecast}/field_table_thompson_aero ./field_table @@ -746,7 +746,7 @@ fi ${NCP} ${PARMforecast}/input.nml.tmp . ${NCP} ${PARMforecast}/input_nest.nml.tmp . ${NCP} ${PARMforecast}/model_configure.tmp . -${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure +${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure # NoahMP table file ${NCP} ${PARMforecast}/noahmptable.tbl . @@ -934,7 +934,7 @@ fi cd .. -# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and nems.configure +# Prepare diag_table, field_table, input.nml, input_nest02.nml, model_configure, and ufs.configure ${NCP} ${PARMforecast}/diag_table.tmp . if [ ${imp_physics:-11} = 8 ]; then ${NCP} ${PARMforecast}/field_table_thompson ./field_table @@ -950,16 +950,16 @@ ${NCP} ${PARMforecast}/noahmptable.tbl . if [ ${ocean_model} = hycom ]; then if [ ${run_ocean} = yes ] || [ ${run_wave} = yes ]; then - ${NCP} ${PARMforecast}/nems.configure.cpl.tmp ./nems.configure.tmp + ${NCP} ${PARMforecast}/ufs.configure.cpl.tmp ./ufs.configure.tmp else - ${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure.tmp + ${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure.tmp fi elif [ ${ocean_model} = mom6 ]; then if [ ${run_ocean} = yes ] || [ ${run_wave} = yes ]; then - ${NCP} ${PARMforecast}/nems.configure.mom6.tmp ./nems.configure.tmp + ${NCP} ${PARMforecast}/ufs.configure.mom6.tmp ./ufs.configure.tmp ${NCP} ${PARMforecast}/data_table ./ else - ${NCP} ${PARMforecast}/nems.configure.atmonly ./nems.configure.tmp + ${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure.tmp fi else echo "WARNING: unknow ocean model of ${ocean_model}" @@ -990,7 +990,7 @@ sed -e "s/_EARTH_component_list_/${EARTH_component_list}/g" \ -e "/_mesh_atm_/d" \ -e "s/_mesh_wav_/ww3_mesh.nc/g" \ -e "s/_multigrid_/false/g" \ - nems.configure.tmp > nems.configure + ufs.configure.tmp > ufs.configure ngrids=${nest_grids} n=1 @@ -1230,10 +1230,10 @@ if [ ${run_datm} = yes ]; then sed -i "s/_yearFirst_/$yr/g" datm.streams sed -i "s/_yearLast_/$endyr/g" datm.streams sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm.streams - # Generate datm_in and nems.configure from model-independent templates: + # Generate datm_in and ufs.configure from model-independent templates: ${NCP} ${PARMhafs}/cdeps/datm_in . sed -i "s/_mesh_atm_/INPUT\/DATM_ESMF_mesh.nc/g" datm_in - ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ + ${NCP} ${PARMforecast}/ufs.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ @@ -1247,7 +1247,7 @@ if [ ${run_datm} = yes ]; then -e "/_system_type_/d" \ -e "s/_atm_model_/datm/g" \ -e "s/_ocn_model_/hycom/g" \ - nems.configure.cdeps.tmp > nems.configure + ufs.configure.cdeps.tmp > ufs.configure elif [ ${run_docn} = yes ]; then MAKE_MESH_OCN=$( echo "${make_mesh_ocn:-no}" | tr a-z A-Z ) ${NLN} "$docn_input_path"/DOCN_input*nc INPUT/ @@ -1272,7 +1272,7 @@ elif [ ${run_docn} = yes ]; then fi done ${NLN} "${mesh_ocn}" INPUT/DOCN_ESMF_mesh.nc - ${NCP} ${PARMforecast}/nems.configure.cdeps.tmp ./ + ${NCP} ${PARMforecast}/ufs.configure.cdeps.tmp ./ sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ @@ -1286,7 +1286,7 @@ elif [ ${run_docn} = yes ]; then -e "s/_system_type_/ufs/g" \ -e "s/_atm_model_/fv3/g" \ -e "s/_ocn_model_/docn/g" \ - nems.configure.cdeps.tmp > nems.configure + ufs.configure.cdeps.tmp > ufs.configure fi # Generate model_configure @@ -1434,8 +1434,8 @@ if [ ${write_dopost:-.false.} = .true. ]; then fi fi -# Copy the fd_nems.yaml file -${NCP} ${HOMEhafs}/sorc/hafs_forecast.fd/tests/parm/fd_nems.yaml ./ +# Copy the fd_ufs.yaml file +${NCP} ${HOMEhafs}/sorc/hafs_forecast.fd/tests/parm/fd_ufs.yaml ./ # Copy the executable and run the forecast FORECASTEXEC=${FORECASTEXEC:-${EXEChafs}/hafs_forecast.x} diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 28bcc5b3b..ed3b955c1 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -4,19 +4,12 @@ source ./machine-setup.sh > /dev/null 2>&1 if [ $target = wcoss2 ]; then source ../versions/build.ver; fi cwd=$(pwd) -cp hafs_forecast.fd/FV3/ccpp/suites_not_used/suite_FV3_HAFS_v1* hafs_forecast.fd/FV3/ccpp/suites/ -cd hafs_forecast.fd/tests - -if [ $target = hera ]; then target=hera ; fi -if [ $target = orion ]; then target=orion ; fi -if [ $target = jet ]; then target=jet ; fi -if [ $target = cheyenne ]; then target=cheyenne ; fi -if [ $target = wcoss2 ]; then target=wcoss2 ; fi +if [[ $target =~ .*c5 ]] ; then target=gaea-c5 ; fi -app=HAFS-MOM6 -./compile.sh "$target" "-DAPP=$app -DREGIONAL_MOM6=ON -DMOVING_NEST=ON -DFASTER=ON -DCCPP_SUITES=FV3_HAFS_v1_thompson_noahmp_nonsst,FV3_HAFS_v1_thompson_noahmp,FV3_HAFS_v1_thompson_nonsst,FV3_HAFS_v1_thompson,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_thompson_tedmf_gfdlsf,FV3_RRFSv2c3 -D32BIT=ON" hafs_mom6 intel YES NO +cd hafs_forecast.fd/tests app=HAFSW -./compile.sh "$target" "-DAPP=$app -DMOVING_NEST=ON -DFASTER=ON -DCCPP_SUITES=FV3_HAFS_v1_thompson_noahmp_nonsst,FV3_HAFS_v1_thompson_noahmp,FV3_HAFS_v1_thompson_nonsst,FV3_HAFS_v1_thompson,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_thompson_tedmf_gfdlsf,FV3_RRFSv2c3 -D32BIT=ON" hafs_hycom intel YES NO +opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1 -D32BIT=ON" +./compile.sh "$target" "$opts" 32bit intel YES NO exit diff --git a/sorc/build_tools.sh b/sorc/build_tools.sh index 9325755ba..381e0cfb6 100755 --- a/sorc/build_tools.sh +++ b/sorc/build_tools.sh @@ -21,11 +21,19 @@ elif [ $target = wcoss2 ]; then export DM_FC="ftn -static" export DM_F90="ftn -free -static" export DM_CC="cc -static" +elif [[ $target =~ gaea ]]; then + export FC="ftn" + export F90="ftn" + export CC=cc + export DM_FC="ftn" + export DM_F90="ftn" + export DM_CC="cc" else echo "Unknown machine = $target" exit 1 fi - +export NETCDF="${NETCDF:-$netcdf_c_ROOT}" +export HDF5="${NETCDF:-$hdf5_ROOT}" export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdff -lnetcdf" export HDF5_INCLUDE=${HDF5_INCLUDE:-"-I${HDF5_INCLUDES:--I${HDF5}/include}"} diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 67258a934..0989b539a 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 67258a934ebf8e0e2d0cf2c914796f853b6f73e6 +Subproject commit 0989b539aaf5f87e4c140e939c36cdeee08f543c diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index e85b300f7..fbb498c11 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit e85b300f7ad23ef570d3bda8c134cbeed4a7b046 +Subproject commit fbb498c1167c5dacd4d83a6be1ccfb61e5103f76 diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index 9d1622b35..15ae4ccc3 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit 9d1622b3552d762160a20ce79969da2076a84513 +Subproject commit 15ae4ccc3a4cd00aa1be4caa4b5d36dea35100e2 diff --git a/sorc/install_all.sh b/sorc/install_all.sh index 1f16376f3..9132de603 100755 --- a/sorc/install_all.sh +++ b/sorc/install_all.sh @@ -15,8 +15,9 @@ CP='/bin/cp -rp' # install forecast #------------------------------------ $Build_forecast && { - ${CP} hafs_forecast.fd/tests/fv3_hafs_mom6.exe ../exec/hafs_forecast_mom6.x - ${CP} hafs_forecast.fd/tests/fv3_hafs_hycom.exe ../exec/hafs_forecast_hycom.x + ${CP} hafs_forecast.fd/tests/fv3_32bit.exe ../exec/hafs_forecast_hycom.x +# ${CP} hafs_forecast.fd/tests/fv3_hafs_mom6.exe ../exec/hafs_forecast_mom6.x +# ${CP} hafs_forecast.fd/tests/fv3_hafs_hycom.exe ../exec/hafs_forecast_hycom.x } #------------------------------------ diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh index b995aea25..15c7810b5 100755 --- a/sorc/link_fix.sh +++ b/sorc/link_fix.sh @@ -21,6 +21,8 @@ elif [ ${target} == "orion" ]; then FIXROOT=/work/noaa/hwrf/noscrub/hafs-fix-files/hafs-${FIXversion}-fix/fix elif [ ${target} == "jet" ]; then FIXROOT=/lfs4/HFIP/hwrf-data/hafs-fix-files/hafs-${FIXversion}-fix/fix +elif [[ $target =~ gaea ]]; then + FIXROOT=/lustre/f2/dev/Samuel.Trahan/hafs/hafs-fix-files/hafs-${FIXversion}-fix/fix else echo "FATAL ERROR: Unknown site " ${target} exit 1 diff --git a/ush/hafs_pre_job.sh.inc b/ush/hafs_pre_job.sh.inc index f54c8c09d..aceaae3f8 100644 --- a/ush/hafs_pre_job.sh.inc +++ b/ush/hafs_pre_job.sh.inc @@ -73,18 +73,14 @@ elif [[ -d /glade ]] ; then target=cheyenne module purge elif [[ -d /lustre && -d /ncrc ]] ; then - # We are on GAEA. - if ( ! eval module help > /dev/null 2>&1 ) ; then - # We cannot simply load the module command. The GAEA - # /etc/profile modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/profile here. - echo load the module command 1>&2 - source /etc/profile + # You cannot "module purge" on GAEA + if ( head /proc/cpuinfo | grep AuthenticAMD > /dev/null ) ; then + target=gaea_c5 + else + target=gaea fi - target=gaea - module purge + ulimit -s unlimited + export TZ=UTC elif [[ "$(hostname)" =~ "odin" ]] ; then target="odin" else diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc index c50260b9b..b4997d4a1 100644 --- a/ush/hafs_runcmd.sh.inc +++ b/ush/hafs_runcmd.sh.inc @@ -11,8 +11,8 @@ export PURE_OMP_THREADS=${PURE_OMP_THREADS:-${OMP_NUM_THREADS:-24}} export NCNODE=${NCNODE:-$(($NCTSK * $OMP_NUM_THREADS))} export KMP_AFFINITY=disabled export KMP_STACKSIZE=${KMP_STACKSIZE:-2048m} -#export OMP_STACKSIZE=${OMP_STACKSIZE:-2048m} -# Avoild "NetCDF: HDF error" +export OMP_STACKSIZE=${OMP_STACKSIZE:-2048m} +# Avoid "NetCDF: HDF error" export HDF5_USE_FILE_LOCKING=FALSE # Set job run commands @@ -41,6 +41,15 @@ elif [ "$machine" = jet ] || [ "$machine" = hera ] || [ "$machine" = orion ]; th export APRUNF="time" export APRUNCFP="time" export BACKGROUND="&" +elif [ "$machine" = gaea_c5 ] ; then + export APRUN="time" + export APRUNS="srun -l --mem=0 --exclusive --ntasks=1 --ntasks-per-node=1 --cpus-per-task=128 " + export APRUNO="srun -l --mem=0 --exclusive --ntasks=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS} -- " + export APRUNC="srun -l --mem=0 --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS} -- " + export APRUNM="$APRUNC" + export APRUNF="$APRUNC ${MPISERIAL:-mpiserial} -m " + export APRUNCFP="$APRUN" + export BACKGROUND="" else echo "WARNING: unknown platform. Guessing the job run commands..." export APRUN="time" diff --git a/ush/produtil/cluster.py b/ush/produtil/cluster.py index b8ac04e6f..dac610d7a 100644 --- a/ush/produtil/cluster.py +++ b/ush/produtil/cluster.py @@ -99,7 +99,12 @@ def where(): elif os.path.exists('/lfs/h2/emc'): here=WCOSS2() elif os.path.exists('/lustre/f2'): - here=NOAAGAEA() + with open("/proc/cpuinfo", "rt") as fd: + is_amd=fd.read(3000).find('EPYC') >= 0 + if is_amd: + here=NOAAGAEAC5() + else: + here=NOAAGAEA() else: here=Cluster(False,False,False,'noname','noname') return here @@ -178,6 +183,14 @@ def __init__(self): super(NOAAGAEA,self).__init__(False,True,False,'gaea', 'gaea.rdhpcs.noaa.gov') +class NOAAGAEAC5(Cluster): + """!Represents the NOAA GAEA cluster C5 partition. Allows ACLs to be used for + restricted data, and specifies that group quotas are not in use.""" + def __init__(self): + """!constructor for NOAAGAEAC5""" + super(NOAAGAEAC5,self).__init__(False,True,False,'gaea_c5', + 'gaea.rdhpcs.noaa.gov') + class NOAAHera(Cluster): """!Represents the NOAA Hera cluster. Does not allow ACLs, assumes no group quotas (fileset quotas instead).""" From 64253d95e3b2578a896b58abaced1f750a69d09b Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 21 Nov 2023 18:58:39 -0500 Subject: [PATCH 02/30] update forecast hash --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index d972f5a8a..892b74d3e 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit d972f5a8a13d075f09294c59f156844c63e3df57 +Subproject commit 892b74d3e2b23c8c5ab44aa6ae9254e22bd8e800 From c5d77d3ff55359f71147182997f9c922ba22bf63 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 21 Nov 2023 19:25:28 -0500 Subject: [PATCH 03/30] update to top of hafs_forecast.fd --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 892b74d3e..08f842d29 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 892b74d3e2b23c8c5ab44aa6ae9254e22bd8e800 +Subproject commit 08f842d292f3b08b767220bdeee4964504cac330 From 0630ebbea99c1fb9e76faa1879b7d07d13f29d4f Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 20:08:44 +0000 Subject: [PATCH 04/30] add old gwd suite --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 08f842d29..9a952bfd4 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 08f842d292f3b08b767220bdeee4964504cac330 +Subproject commit 9a952bfd4e572233e6464f0e41b12b63b87052d1 From 1726ec6dd4a17de780ff4432363bc19329abb923 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 20:10:25 +0000 Subject: [PATCH 05/30] switch to spack stack --- modulefiles/hafs.hera.lua | 163 ++++++------------ .../sorc/hafs_obs_preproc/CMakeLists.txt | 2 +- 2 files changed, 54 insertions(+), 111 deletions(-) diff --git a/modulefiles/hafs.hera.lua b/modulefiles/hafs.hera.lua index d808b6501..4bc866b73 100644 --- a/modulefiles/hafs.hera.lua +++ b/modulefiles/hafs.hera.lua @@ -6,122 +6,65 @@ prepend_path("MODULEPATH", "/contrib/sutils/modulefiles") load("sutils") load("hpss") -cmake_ver=os.getenv("cmake_ver") or "3.20.1" -load(pathJoin("cmake", cmake_ver)) - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles") -load(pathJoin("python","wcoss2_env")) - -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack") -hpc_ver=os.getenv("hpc_ver") or "1.2.0" -load(pathJoin("hpc", hpc_ver)) - -hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2" -load(pathJoin("hpc-intel", hpc_intel_ver)) - -hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2" -load(pathJoin("hpc-impi", hpc_impi_ver)) - -jasper_ver=os.getenv("jasper_ver") or "2.0.25" -load(pathJoin("jasper", jasper_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.11" -load(pathJoin("zlib", zlib_ver)) - -zstd_ver=os.getenv("zstd_ver") or "1.5.0" -load(pathJoin("zstd", zstd_ver)) - -libpng_ver=os.getenv("libpng_ver") or "1.6.37" -load(pathJoin("libpng", libpng_ver)) - -hdf5_ver=os.getenv("hdf5_ver") or "1.10.6" -load(pathJoin("hdf5", hdf5_ver)) - -netcdf_ver=os.getenv("netcdf_ver") or "4.7.4" -load(pathJoin("netcdf", netcdf_ver)) - -pio_ver=os.getenv("pio_ver") or "2.5.7" -load(pathJoin("pio", pio_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.3.0b09" -load(pathJoin("esmf", esmf_ver)) - -fms_ver=os.getenv("fms_ver") or "2022.04" -load(pathJoin("fms",fms_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" -load(pathJoin("bacio", bacio_ver)) - -crtm_ver=os.getenv("crtm_ver") or "2.4.0" -load(pathJoin("crtm", crtm_ver)) - -g2_ver=os.getenv("g2_ver") or "3.4.5" -load(pathJoin("g2", g2_ver)) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") -g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2" -load(pathJoin("g2tmpl", g2tmpl_ver)) +stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +load(pathJoin("stack-intel", stack_intel_ver)) -ip_ver=os.getenv("ip_ver") or "3.3.3" -load(pathJoin("ip", ip_ver)) +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) -nemsio_ver_ver=os.getenv("nemsio_ver") or "2.5.4" -load(pathJoin("nemsio", nemsio_ver)) - -sp_ver=os.getenv("sp_ver") or "2.3.3" -load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" -load(pathJoin("w3emc", w3emc_ver)) - -w3nco_ver=os.getenv("w3nco_ver") or "2.4.1" -load(pathJoin("w3nco", w3nco_ver)) - -gftl_shared_ver=os.getenv("gftl_shared_ver") or "v1.5.0" -load(pathJoin("gftl-shared", gftl_shared_ver)) - -yafyaml_ver=os.getenv("yafyaml_ver") or "v0.5.1" -load(pathJoin("yafyaml", yafyaml_ver)) - -mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09" -load(pathJoin("mapl", mapl_ver)) - -bufr_ver=os.getenv("bufr_ver") or "11.4.0" -load(pathJoin("bufr", bufr_ver)) - -sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" -load(pathJoin("sfcio", sfcio_ver)) - -sigio_ver=os.getenv("sigio_ver") or "2.3.2" -load(pathJoin("sigio", sigio_ver)) - -szip_ver=os.getenv("szip_ver") or "2.1.1" -load(pathJoin("szip", szip_ver)) - -wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" -load(pathJoin("wrf_io", wrf_io_ver)) - -prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" -load(pathJoin("prod_util", prod_util_ver)) - -grib_util_ver=os.getenv("grib_util_ver") or "1.2.4" -load(pathJoin("grib_util", grib_util_ver)) - -wgrib2_ver=os.getenv("wgrib2_ver") or "2.0.8" -load(pathJoin("wgrib2", wgrib2_ver)) -setenv("WGRIB2", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/impi-2018.0.4/wgrib2/2.0.8/bin/wgrib2") - -gempak_ver=os.getenv("gempak_ver") or "7.4.2" -load(pathJoin("gempak", gempak_ver)) +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) --- nco_ver=os.getenv("nco_ver") or "5.0.6" -nco_ver=os.getenv("nco_ver") or "4.9.3" -load(pathJoin("nco", nco_ver)) +local hafs_modules = { + {["jasper"] = "2.0.32"}, + {["zlib"] = "1.2.13"}, + {["zstd"] = "1.5.2"}, + {["libpng"] = "1.6.37"}, + {["hdf5"] = "1.14.0"}, + {["netcdf-c"] = "4.9.2"}, + {["netcdf-fortran"] = "4.6.0"}, + {["parallelio"] = "2.5.10"}, + {["esmf"] = "8.4.2"}, + {["fms"] = "2023.01"}, + {["bacio"] = "2.4.1"}, + {["crtm"] = "2.4.0"}, + {["g2"] = "3.4.5"}, + {["g2tmpl"] = "1.10.2"}, + {["ip"] = "4.3.0"}, + {["nemsio"] = "2.5.4"}, + {["sp"] = "2.3.3"}, + {["w3emc"] = "2.10.0"}, + {["w3nco"] = "2.4.1"}, + {["gftl-shared"] = "1.5.0"}, + {["yafyaml"] = "0.5.1"}, + {["mapl"] = "2.35.2-esmf-8.4.2"}, + {["bufr"] = "12.0.0"}, + {["sfcio"] = "1.4.1"}, + {["sigio"] = "2.3.2"}, + {["szip"] = "2.1"}, + {["wrf-io"] = "1.2.0"}, + {["prod_util"] = "1.2.2"}, + {["grib-util"] = "1.3.0"}, + {["wgrib2"] = "2.0.8"}, + {["gempak"] = "7.4.2"}, + {["nco"] = "5.0.6"}, + {["cdo"] = "2.0.5"}, + {["rocoto"] = "1.3.3"}, + {["scotch"] = "7.0.4"}, +} + +for i = 1, #hafs_modules do + for name, default_version in pairs(hafs_modules[i]) do + local env_version_name = string.gsub(name, "-", "_") .. "_ver" + load(pathJoin(name, os.getenv(env_version_name) or default_version)) + end +end -cdo_ver=os.getenv("cdo_ver") or "1.9.8" -load(pathJoin("cdo", cdo_ver)) -rocoto_ver=os.getenv("rocoto_ver") or "1.3.3" -load(pathJoin("rocoto", rocoto_ver)) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/hwrf/noscrub/local/modulefiles") +load(pathJoin("python","wcoss2_env")) setenv("CMAKE_C_COMPILER", "mpiicc") setenv("CMAKE_CXX_COMPILER", "mpiicpc") diff --git a/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt b/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt index 7f8625d78..9aa5684c1 100644 --- a/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt +++ b/sorc/hafs_tools.fd/sorc/hafs_obs_preproc/CMakeLists.txt @@ -87,7 +87,7 @@ target_link_options(${exe_name} PRIVATE target_link_libraries( ${exe_name} - bufr::bufr_d + bufr::bufr_4 NetCDF::NetCDF_Fortran NetCDF::NetCDF_C MPI::MPI_Fortran) From 100f6998b2c95ff561b5862e13a9479ca690cdf0 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 20:10:54 +0000 Subject: [PATCH 06/30] use BUFR_LIB4 so hera spack stack will compile --- sorc/build_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_tools.sh b/sorc/build_tools.sh index 381e0cfb6..164fb6628 100755 --- a/sorc/build_tools.sh +++ b/sorc/build_tools.sh @@ -38,7 +38,7 @@ export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdff -lnetcdf" export HDF5_INCLUDE=${HDF5_INCLUDE:-"-I${HDF5_INCLUDES:--I${HDF5}/include}"} export HDF5_LDFLAGS=${HDF5_LDFLAGS:-"-L${HDF5_LIBRARIES:-${HDF5}/lib} -lhdf5_hl -lhdf5"} -export BUFR_LDFLAGS="${BUFR_LIBd}" +export BUFR_LDFLAGS="${BUFR_LIB4}" TOOLS_PATH=${cwd}/hafs_tools.fd export TOOLS_INC=${TOOLS_PATH}/include From a6ba20801ccb2c992a0002efc75757b529f76ccf Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 20:11:09 +0000 Subject: [PATCH 07/30] include old gwd suite too --- sorc/build_forecast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index ed3b955c1..6423debf5 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -9,7 +9,7 @@ if [[ $target =~ .*c5 ]] ; then target=gaea-c5 ; fi cd hafs_forecast.fd/tests app=HAFSW -opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1 -D32BIT=ON" +opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1,FV3_global_nest_v0 -D32BIT=ON" ./compile.sh "$target" "$opts" 32bit intel YES NO exit From d43e206bd0b74eb5d83bec2e15f4b4bc4e3e038c Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 20:12:09 +0000 Subject: [PATCH 08/30] use old GWD suite --- parm/WaterInTheWest.conf | 6 +++--- parm/WaterInTheWestC192.conf | 6 +++--- parm/WaterInTheWestC384.conf | 6 +++--- parm/WaterInTheWestC96.conf | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/parm/WaterInTheWest.conf b/parm/WaterInTheWest.conf index 1f1968277..5eea21450 100644 --- a/parm/WaterInTheWest.conf +++ b/parm/WaterInTheWest.conf @@ -69,9 +69,9 @@ jend_nest=1294 regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v1 -ccpp_suite_glob=FV3_global_nest_v1 -ccpp_suite_nest=FV3_global_nest_v1 +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 diff --git a/parm/WaterInTheWestC192.conf b/parm/WaterInTheWestC192.conf index 4a43917bf..ce522f99a 100644 --- a/parm/WaterInTheWestC192.conf +++ b/parm/WaterInTheWestC192.conf @@ -69,9 +69,9 @@ jend_nest=322 regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v1 -ccpp_suite_glob=FV3_global_nest_v1 -ccpp_suite_nest=FV3_global_nest_v1 +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 diff --git a/parm/WaterInTheWestC384.conf b/parm/WaterInTheWestC384.conf index e1fc616d4..947860e7c 100644 --- a/parm/WaterInTheWestC384.conf +++ b/parm/WaterInTheWestC384.conf @@ -69,9 +69,9 @@ jend_nest=648 regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v1 -ccpp_suite_glob=FV3_global_nest_v1 -ccpp_suite_nest=FV3_global_nest_v1 +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 diff --git a/parm/WaterInTheWestC96.conf b/parm/WaterInTheWestC96.conf index 8d37d9e2d..916db6a4e 100644 --- a/parm/WaterInTheWestC96.conf +++ b/parm/WaterInTheWestC96.conf @@ -69,9 +69,9 @@ jend_nest=160 regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v1 -ccpp_suite_glob=FV3_global_nest_v1 -ccpp_suite_nest=FV3_global_nest_v1 +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 # For the global domain if it is a global or global-nesting experiment glob_k_split=1 From c72ffd8f61cc635a815657ea412d0137528f30a6 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 22 Nov 2023 16:44:20 -0500 Subject: [PATCH 09/30] no cray-libsci --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 08f842d29..1a4e3b2ac 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 08f842d292f3b08b767220bdeee4964504cac330 +Subproject commit 1a4e3b2aca9cb46c57ab29aad90081fb0dcc119f From 535639ef804bc28ed866178e548c461ffa0948dd Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 22 Nov 2023 17:50:32 -0500 Subject: [PATCH 10/30] compile old gwd --- sorc/build_forecast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index ed3b955c1..6423debf5 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -9,7 +9,7 @@ if [[ $target =~ .*c5 ]] ; then target=gaea-c5 ; fi cd hafs_forecast.fd/tests app=HAFSW -opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1 -D32BIT=ON" +opts="-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_global_nest_v1,FV3_global_nest_v0 -D32BIT=ON" ./compile.sh "$target" "$opts" 32bit intel YES NO exit From dd853abb2590682ad986164f7de67044b01178a9 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Wed, 22 Nov 2023 23:25:28 +0000 Subject: [PATCH 11/30] revert rocoto script changes --- rocoto/WaterInTheWest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh index 44dec0b7b..f28182fd7 100755 --- a/rocoto/WaterInTheWest.sh +++ b/rocoto/WaterInTheWest.sh @@ -7,7 +7,7 @@ set -x cd ${HOMEhafs}/rocoto EXPT=$(basename ${HOMEhafs}) -SUBEXPT=C768 +SUBEXPT=$EXPT conf=../parm/WaterInTheWest.conf opts="-t -f" scrubopt="config.scrub_work=no config.scrub_com=no" From 5a890a67aaa066431a22ec867e8bf8e42be3d334 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Fri, 1 Dec 2023 01:06:01 +0000 Subject: [PATCH 12/30] gtype=stretched works --- jobs/JHAFS_PRODUCT | 2 +- parm/WaterInTheWestC384NoNest.conf | 176 ++++++++ parm/forecast/globnest/input.nml.nonest.tmp | 421 ++++++++++++++++++++ scripts/exhafs_atm_post.sh | 4 +- scripts/exhafs_forecast.sh | 14 +- 5 files changed, 609 insertions(+), 8 deletions(-) create mode 100644 parm/WaterInTheWestC384NoNest.conf create mode 100644 parm/forecast/globnest/input.nml.nonest.tmp diff --git a/jobs/JHAFS_PRODUCT b/jobs/JHAFS_PRODUCT index 8de58a716..2eaffd7d8 100755 --- a/jobs/JHAFS_PRODUCT +++ b/jobs/JHAFS_PRODUCT @@ -70,7 +70,7 @@ fi mkdir -p $DATA cd $DATA -if [ ${gtype} = nest ]; then +if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then ngrids=$((${nest_grids} + 1)) else ngrids=${nest_grids} diff --git a/parm/WaterInTheWestC384NoNest.conf b/parm/WaterInTheWestC384NoNest.conf new file mode 100644 index 000000000..ffa22dbd1 --- /dev/null +++ b/parm/WaterInTheWestC384NoNest.conf @@ -0,0 +1,176 @@ +# This is a UNIX conf file that contains ALL information relating to +# the HAFS basin-focused global-nesting configuration. +# +[config] +# Specify the domain center Latitude and Longitude +domlat=32.5 +domlon=-135.0 +# Same as domlat and domlon but vitals formatted +vitalsdomlat=325N +vitalsdomlon=1350W + +# HAFS launcher requires vitals and a storm. This is a placeholder for the global static nest. +fakestormid=00L +fakestormname=NATL +fakestormorg=NHC +fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M + +run_atm_mvnest=no +run_wave=no +ww3_model=ww3 +run_ocean=no +ocean_model=hycom + +run_atm_init=no +run_atm_init_fgat=no +run_atm_init_ens=no +run_atm_merge=no +run_atm_merge_fgat=no +run_atm_merge_ens=no +run_atm_vi=no +run_atm_vi_fgat=no +run_atm_vi_ens=no +run_gsi=no +gsi_d01=no +gsi_d02=no +run_analysis_merge=no +run_analysis_merge_ens=no +run_fgat=no +run_envar=no +run_ensda=no +run_enkf=no + +[dir] +PARMforecast={PARMhafs}/forecast/globnest + +[grid] +CASE=C384 ;; FV3 resolution +LEVS=65 ;; Model vertical levels: 65 +gtype=stretch ;; grid type: uniform, stretch, nest, or regional +# If gridfixdir is provided and the dir exists, then use the pre-generated static grid fix files under gridfixdir +#gridfixdir=/let/hafs_grid/generate/grid +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/HAFS-weird-C768 +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest +# Otherwise, will generate the model grid according to the following grid parameters +# Need for grid types: stretch, nest and regional +stretch_fac=1.0001 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +# Need for grid types: nest and regional +nest_grids=0 +parent_grid_num=1 +parent_tile=6 +refine_ratio=4 +istart_nest=25 +jstart_nest=73 +iend_nest=744 +jend_nest=648 +regional_esg=no + +[forecast] +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 + +# For the global domain if it is a global or global-nesting experiment +glob_k_split=1 +glob_n_split=7 +glob_layoutx=10 +glob_layouty=10 +glob_npx=385 +glob_npy=385 +glob_io_layoutx=1 +glob_io_layouty=1 +glob_full_zs_filter=.true. +glob_n_zs_filter=0 +glob_n_del2_weak=15 +glob_max_slope=0.15 +glob_shal_cnv=.true. +glob_do_deep=.true. +glob_imfshalcnv=5 +glob_imfdeepcnv=5 +# For the nested or regional standalone domain +k_split=4 +n_split=5 +layoutx=34 +layouty=24 +npx=1441 +npy=961 +io_layoutx=1 +io_layouty=1 +full_zs_filter=.true. +n_zs_filter=1 +n_del2_weak=24 +max_slope=0.15 +shal_cnv=.true. +do_deep=.true. + +npz=64 + +is_moving_nest=.false. + +quilting=.true. +write_dopost=.true. +write_tasks_per_group=120 +output_history=.true. + +output_grid=regional_latlon,regional_latlon +output_grid_cen_lon=180.0,{domlon} ;; central longitude +output_grid_cen_lat=0.0,{domlat} ;; central latitude +output_grid_lon_span=360.0,120.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=180.0,80.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.25,0.0625 ;; output grid spacing dlon . . . . +output_grid_dlat=0.25,0.0625 ;; output grid spacing dlat . . . . + +dt_atmos=120 + +# Thompson options +imp_physics=8 +iovr=3 +dt_inner=60 +dnats=0 +do_sat_adj=.false. +lgfdlmprad=.false. +nstf_n1=2 + +# Other RRFS physics options. +iaer=5111 +ltaerosol=.true. +satmedmf=.false. +do_mynnedmf=.true. +do_mynnsfclay=.true. +cdmbgwd=0.14,1.8,1.0,1.0 +iopt_sfc=4 +gwd_opt=2 +do_ugwp_v0=.true. +do_ugwp_v1=.false. +do_ugwp_v0_orog_only=.false. +do_ugwp_v0_nst_only=.false. +do_ugwp_v1_w_gsldrag=.false. +do_ugwp_v1_orog_only=.false. +do_gsl_drag_ls_bl=.false. +do_gsl_drag_ss=.false. +do_gsl_drag_tofd=.false. +bl_mynn_tkeadvect = .true. +lsm=2 + +# Diagnostics +lightning_threat=.true. + + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} +postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt +postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_840PE +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES diff --git a/parm/forecast/globnest/input.nml.nonest.tmp b/parm/forecast/globnest/input.nml.nonest.tmp new file mode 100644 index 000000000..27bec3441 --- /dev/null +++ b/parm/forecast/globnest/input.nml.nonest.tmp @@ -0,0 +1,421 @@ +&atmos_model_nml + blocksize = @[blocksize] + chksum_debug = .false. + dycore_only = .false. + avg_max_length = 10800. + ccpp_suite = '@[ccpp_suite_nml]' + ignore_rst_cksum = .true. +/ + +&diag_manager_nml + prepend_date = .false. +/ + +&mpp_io_nml + header_buffer_val = 16384, + global_field_on_root_pe = .true., + io_clocks_on = .false., + shuffle = 0, + deflate_level = @[deflate_level], + cf_compliance = .false. +/ + +&fms_io_nml + checksum_required = .false. + max_files_r = 1000, + max_files_w = 1000, +/ + +&fms2_io_nml + netcdf_default_format="netcdf4" +/ + +&fms_nml + clock_grain = 'ROUTINE', + domains_stack_size = 30000000, + print_memory_usage = .false. +/ + +&fv_grid_nml + !grid_file = 'INPUT/grid_spec.nc' +/ + +&fv_core_nml + ignore_rst_cksum = .true. + layout = @[layoutx_nml],@[layouty_nml] + io_layout = @[io_layoutx_nml],@[io_layouty_nml] + npx = @[npx_nml] + npy = @[npy_nml] + ntiles = 6 + npz = @[npz] + !grid_type = -1 + make_nh = .false. + fv_debug = .false. + range_warn = .true. + reset_eta = .false. + upoff = 2 + n_sponge = 24 + nudge_qv = .false. + nudge_dz = .false. + tau = 5. + rf_cutoff = 30.e2 + sg_cutoff = -1. + rf_fast = .false. + d2_bg_k1 = 0.16 + d2_bg_k2 = 0.05 + kord_tm = -9 + kord_mt = 9 + kord_wz = 9 + kord_tr = 9 + hydrostatic = .false. + phys_hydrostatic = .false. + use_hydro_pressure = .false. + beta = 0. + a_imp = 1. + p_fac = 0.1 + k_split = @[k_split_nml] + n_split = @[n_split_nml] + nwat = 6 + na_init = @[na_init] + d_ext = 0.0 + dnats = @[dnats] + fv_sg_adj = 300 + d2_bg = 0. + nord = 2 + dddmp = 0.1 + d4_bg = 0.15 + vtdm4 = 0.04 + delt_max = 0.002 + dz_min = 2.0 + ke_bg = 0. + do_vort_damp = .true. + external_ic = @[external_ic] + external_eta = .true. + gfs_phil = .false. + nggps_ic = @[nggps_ic] + mountain = @[mountain] + ncep_ic = .false. + d_con = 1.0 + hord_mt = 5 + hord_vt = 5 + hord_tm = 5 + hord_dp = -5 + hord_tr = -5 + adjust_dry_mass = .false. + consv_te = 1. + do_sat_adj = @[do_sat_adj] + consv_am = .false. + fill = .true. + dwind_2d = .false. + print_freq = 3 + warm_start = @[warm_start] + no_dycore = .false. + z_tracer = .true. + agrid_vel_rst = .true. + read_increment = .false. + res_latlon_dynamics = "fv3_increment.nc" + write_3d_diags = .true. + + do_schmidt = .true. + target_lat = @[target_lat] + target_lon = @[target_lon] + stretch_fac = @[stretch_fac] + + full_zs_filter = @[full_zs_filter_nml] + n_zs_filter = @[n_zs_filter_nml] + nord_zs_filter = 4 +/ + + +&surf_map_nml + zero_ocean = .false. + cd4 = 0.15 + cd2 = -1 + n_del2_strong = 0 + n_del2_weak = @[n_del2_weak_nml] + n_del4 = 2 + max_slope = @[max_slope_nml] + peak_fac = 1. +/ + +&external_ic_nml + filtered_terrain = .true. + levp = @[levp] + gfs_dwinds = .true. + checker_tr = .false. + nt_checker = 0 +/ + +&gfs_physics_nml + lightning_threat = @[lightning_threat] + fhzero = 3. + ldiag3d = .false. + lradar = .true. + avg_max_length = 10800. + h2o_phys = .true. + fhcyc = 0. + use_ufo = .true. + pre_rad = .false. + imp_physics = @[imp_physics] + iovr = @[iovr] + ltaerosol = @[ltaerosol] + ttendlim = -999 + dt_inner = @[dt_inner] + sedi_semi = .true. + decfl = 10 + pdfcld = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false + fhswr = @[fhswr] + fhlwr = @[fhlwr] + ialb = @[ialb] + iems = @[iems] + iaer = @[iaer] + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .true. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .false. + hurr_pbl = .true. ! HWRF moninedmf + moninq_fac = -1.0 ! HWRF_moninedmf + do_mynnedmf = @[do_mynnedmf] + do_mynnsfclay = @[do_mynnsfclay] + bl_mynn_tkeadvect = @[bl_mynn_tkeadvect] + satmedmf = @[satmedmf] + isatmedmf = 1 + lseaspray = @[lseaspray] + rlmx = @[rlmx_nml] + elmx = @[elmx_nml] + sfc_rlm = @[sfc_rlm_nml] + tc_pbl = @[tc_pbl_nml] + random_clds = .false. + trans_trac = .true. + cnvcld = .true. + shal_cnv = @[shal_cnv_nml] !Shallow convection + do_deep = @[do_deep_nml] + imfshalcnv = @[imfshalcnv] + imfdeepcnv = @[imfdeepcnv] + clam_deep = @[clam_deep] + betal_deep = 0.05 + betas_deep = 0.05 + cdmbgwd = @[cdmbgwd] + sfc_z0_type = 6 + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + lsoil = 4 + lsm = @[lsm] + iopt_dveg = 4 + iopt_crs = 2 + iopt_btr = 1 + iopt_run = 1 + iopt_sfc = @[iopt_sfc] + iopt_frz = 1 + iopt_inf = 1 + iopt_rad = 3 + iopt_alb = 1 + iopt_snf = 4 + iopt_tbot = 2 + iopt_stc = 3 + iopt_trs = 2 + lsoil_lsm = 4 + debug = .false. + oz_phys = .false. + oz_phys_2015 = .true. + nst_anl = .true. + nstf_name = @[nstf_n1],@[nstf_n2],@[nstf_n3],@[nstf_n4],@[nstf_n5] + cplflx = @[cplflx] + cplice = .false. + cplocn2atm = @[cplocn2atm] + cplwav = @[cplwav] + cplwav2atm = @[cplwav2atm] + cpl_imp_mrg = @[merge_import] + psautco = 0.0008, 0.0005 + prautco = 0.00015, 0.00015 + iau_delthrs = 6 + iaufhrs = 30 + iau_inc_files = '' + iau_drymassfixer = .false. + lgfdlmprad = @[lgfdlmprad] + effr_in = .true. + do_sppt = @[do_sppt] + do_shum = @[do_shum] + do_skeb = @[do_skeb] + lndp_type = 0 + n_var_lndp = 0 + do_ca = .false. + ca_sgs = .false. + nca = 1 + ncells = 5 + nlives = 12 + nseed = 1 + nfracseed = 0.5 + ca_trigger = .false. + ca_entr = .false. + ca_closure = .false. + ca_global = .false. + nca_g = 1 + ncells_g = 1 + nlives_g = 100 + nseed_g = 100 + ca_smooth = .false. + nspinup = 1 + iseed_ca = 1 + nsmooth = 100 + ca_amplitude = 0.35 + print_diff_pgr = .true. + do_ugwp = .false. + do_tofd = .false. + gwd_opt = @[gwd_opt] + do_ugwp_v0 = @[do_ugwp_v0] + do_ugwp_v1 = @[do_ugwp_v1] + do_ugwp_v0_orog_only = @[do_ugwp_v0_orog_only] + do_ugwp_v0_nst_only = @[do_ugwp_v0_nst_only] + do_ugwp_v1_w_gsldrag = @[do_ugwp_v1_w_gsldrag] + do_ugwp_v1_orog_only = @[do_ugwp_v1_orog_only] + do_gsl_drag_ls_bl = @[do_gsl_drag_ls_bl] + do_gsl_drag_ss = @[do_gsl_drag_ss] + do_gsl_drag_tofd = @[do_gsl_drag_tofd] +/ + +&gfdl_cloud_microphysics_nml + sedi_transport = .true. + do_sedi_heat = .false. + rad_snow = .true. + rad_graupel = .true. + rad_rain = .true. + const_vi = .false. + const_vs = .false. + const_vg = .false. + const_vr = .false. + vi_max = 1. + vs_max = 2. + vg_max = 12. + vr_max = 12. + qi_lim = 1. + prog_ccn = .false. + do_qa = .true. + fast_sat_adj = .true. + tau_l2v = 180. + tau_v2l = 90. + tau_g2v = 900. + rthresh = 10.e-6 ! This is a key parameter for cloud water + dw_land = 0.16 + dw_ocean = 0.10 + ql_gen = 1.0e-3 + ql_mlt = 1.0e-3 + qi0_crt = 8.0E-5 + qs0_crt = 1.0e-3 + tau_i2s = 1000. + c_psaci = 0.05 + c_pgacs = 0.01 + rh_inc = 0.30 + rh_inr = 0.30 + rh_ins = 0.30 + ccn_l = 300. + ccn_o = 100. + c_paut = 0.5 + c_cracw = 0.8 + use_ppm = .false. + use_ccn = .true. + mono_prof = .true. + z_slope_liq = .true. + z_slope_ice = .true. + de_ice = .false. + fix_negative = .true. + icloud_f = 1 + mp_time = 90. +/ + +&interpolator_nml + interp_method = 'conserve_great_circle' +/ + +&namsfc + FNGLAC = "global_glacier.2x2.grb", + FNMXIC = "global_maxice.2x2.grb", + FNTSFC = "RTGSST.1982.2012.monthly.clim.grb", + FNSNOC = "global_snoclim.1.875.grb", + FNZORC = "igbp" + !FNZORC = "global_zorclim.1x1.grb", + FNALBC = "global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb", + FNALBC2 = "global_albedo4.1x1.grb", + FNAISC = "CFSR.SEAICE.1982.2012.monthly.clim.grb", + FNTG3C = "global_tg3clim.2.6x1.5.grb", + FNVEGC = "global_vegfrac.0.144.decpercent.grb", + FNVETC = "global_vegtype.igbp.t1534.3072.1536.rg.grb", + FNSOTC = "global_soiltype.statsgo.t1534.3072.1536.rg.grb", + FNSMCC = "global_soilmgldas.t1534.3072.1536.grb", + FNMSKH = "seaice_newland.grb", + FNTSFA = "", + FNACNA = "", + FNSNOA = "", + FNVMNC = "global_shdmin.0.144x0.144.grb", + FNVMXC = "global_shdmax.0.144x0.144.grb", + FNSLPC = "global_slope.1x1.grb", + FNABSC = "global_mxsnoalb.uariz.t1534.3072.1536.rg.grb", + LDEBUG = .true., + FSMCL(2) = 99999 + FSMCL(3) = 99999 + FSMCL(4) = 99999 + FTSFS = 90 + FAISS = 99999 + FSNOL = 99999 + FSICL = 99999 + FTSFL = 99999 + FAISL = 99999 + FVETL = 99999, + FSOTL = 99999, + FvmnL = 99999, + FvmxL = 99999, + FSLPL = 99999, + FABSL = 99999, + FSNOS = 99999, + FSICS = 99999, +/ + +&nam_stochy + skebnorm = 1 + skeb_npass = 30 + skeb_vdof = 5 + skeb = 0.6,-999,-999,-999,-999 + skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7 + skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + shum = 0.003 + shum_tau = 21600 + shum_lscale = 500000 + sppt = 0.4,0.2,0.1,0.04,0.02 + sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7 + sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + sppt_logit = .true. + sppt_sfclimit = .true. + use_zmtnblck = .true. + iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5] + iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5] + iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5] +/ + +&nam_sfcperts +/ + +&cires_ugwp_nml + knob_ugwp_solver = 2 + knob_ugwp_source = 1,1,0,0 + knob_ugwp_wvspec = 1,25,25,25 + knob_ugwp_azdir = 2,4,4,4 + knob_ugwp_stoch = 0,0,0,0 + knob_ugwp_effac = 1,1,1,1 + knob_ugwp_doaxyz = 1 + knob_ugwp_doheat = 1 + knob_ugwp_dokdis = 1 + knob_ugwp_ndx4lh = 1 + knob_ugwp_version = 0 + launch_level = 25 +/ diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index 7594d8809..8c377c5d2 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -89,7 +89,7 @@ if [ "${POST_CLEANUP^^}" = "YES" ]; then FHR3=$(printf "%03d" "$FHR") # Loop for forecast hours while [ $FHR -le $NHRS ]; do - if [ ${gtype} = nest ]; then + if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then ngrids=$((${nest_grids} + 1)) else ngrids=${nest_grids} @@ -132,7 +132,7 @@ MM=$(echo $NEWDATE | cut -c5-6) DD=$(echo $NEWDATE | cut -c7-8) HH=$(echo $NEWDATE | cut -c9-10) -if [ ${gtype} = nest ]; then +if [ ${gtype} = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then ngrids=$((${nest_grids} + 1)) else ngrids=${nest_grids} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index a39ad9864..689ee1069 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -347,7 +347,7 @@ if [ $gtype = regional ]; then if [ $quilting = .true. ]; then ATM_tasks=$(($ATM_tasks+$write_groups*$write_tasks_per_group)) fi -elif [ $gtype = nest ]; then +elif [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then ATM_tasks=$(( ${glob_layoutx} * ${glob_layouty} * 6 )) for n in $(seq 1 ${nest_grids}); do layoutx_tmp=$( echo ${layoutx} | cut -d , -f ${n} ) @@ -682,7 +682,7 @@ if [ ${imp_physics:-11} = 8 ]; then ${NLN} ${FIXam}/freezeH2O.dat ./ fi -if [ $gtype = nest ]; then +if [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then cd ./INPUT @@ -743,8 +743,12 @@ if [ ${imp_physics:-11} = 8 ]; then else ${NCP} ${PARMforecast}/field_table . fi -${NCP} ${PARMforecast}/input.nml.tmp . -${NCP} ${PARMforecast}/input_nest.nml.tmp . +if [ $gtype = stretch ] || [ $gtype = uniform ]; then + ${NCP} ${PARMforecast}/input.nml.nonest.tmp input.nml.tmp +else + ${NCP} ${PARMforecast}/input.nml.tmp . + ${NCP} ${PARMforecast}/input_nest.nml.tmp . +fi ${NCP} ${PARMforecast}/model_configure.tmp . ${NCP} ${PARMforecast}/ufs.configure.atmonly ./ufs.configure @@ -1315,7 +1319,7 @@ OUTPUT_FH=-1 if [ $gtype = regional ]; then ngrids=${nest_grids} -elif [ $gtype = nest ]; then +elif [ $gtype = nest ] || [ $gtype = stretch ] || [ $gtype = uniform ]; then ngrids=$(( ${nest_grids} + 1 )) else echo "FATAL ERROR: Unsupported gtype of ${gtype}. Currently onnly support gtype of nest or regional." From 357be647abf6475542cea628a79923194c757c68 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Mon, 4 Dec 2023 21:56:42 +0000 Subject: [PATCH 13/30] uniform c384 case --- parm/WaterInTheWestC384Uniform.conf | 176 ++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 parm/WaterInTheWestC384Uniform.conf diff --git a/parm/WaterInTheWestC384Uniform.conf b/parm/WaterInTheWestC384Uniform.conf new file mode 100644 index 000000000..0eb365aaf --- /dev/null +++ b/parm/WaterInTheWestC384Uniform.conf @@ -0,0 +1,176 @@ +# This is a UNIX conf file that contains ALL information relating to +# the HAFS basin-focused global-nesting configuration. +# +[config] +# Specify the domain center Latitude and Longitude +domlat=32.5 +domlon=-135.0 +# Same as domlat and domlon but vitals formatted +vitalsdomlat=325N +vitalsdomlon=1350W + +# HAFS launcher requires vitals and a storm. This is a placeholder for the global static nest. +fakestormid=00L +fakestormname=NATL +fakestormorg=NHC +fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M + +run_atm_mvnest=no +run_wave=no +ww3_model=ww3 +run_ocean=no +ocean_model=hycom + +run_atm_init=no +run_atm_init_fgat=no +run_atm_init_ens=no +run_atm_merge=no +run_atm_merge_fgat=no +run_atm_merge_ens=no +run_atm_vi=no +run_atm_vi_fgat=no +run_atm_vi_ens=no +run_gsi=no +gsi_d01=no +gsi_d02=no +run_analysis_merge=no +run_analysis_merge_ens=no +run_fgat=no +run_envar=no +run_ensda=no +run_enkf=no + +[dir] +PARMforecast={PARMhafs}/forecast/globnest + +[grid] +CASE=C384 ;; FV3 resolution +LEVS=65 ;; Model vertical levels: 65 +gtype=uniform ;; grid type: uniform, stretch, nest, or regional +# If gridfixdir is provided and the dir exists, then use the pre-generated static grid fix files under gridfixdir +#gridfixdir=/let/hafs_grid/generate/grid +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/HAFS-weird-C768 +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest +#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest +# Otherwise, will generate the model grid according to the following grid parameters +# Need for grid types: stretch, nest and regional +stretch_fac=1.0001 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +# Need for grid types: nest and regional +nest_grids=0 +parent_grid_num=1 +parent_tile=6 +refine_ratio=4 +istart_nest=25 +jstart_nest=73 +iend_nest=744 +jend_nest=648 +regional_esg=no + +[forecast] +ccpp_suite_regional=FV3_global_nest_v0 +ccpp_suite_glob=FV3_global_nest_v0 +ccpp_suite_nest=FV3_global_nest_v0 + +# For the global domain if it is a global or global-nesting experiment +glob_k_split=1 +glob_n_split=7 +glob_layoutx=10 +glob_layouty=10 +glob_npx=385 +glob_npy=385 +glob_io_layoutx=1 +glob_io_layouty=1 +glob_full_zs_filter=.true. +glob_n_zs_filter=0 +glob_n_del2_weak=15 +glob_max_slope=0.15 +glob_shal_cnv=.true. +glob_do_deep=.true. +glob_imfshalcnv=5 +glob_imfdeepcnv=5 +# For the nested or regional standalone domain +k_split=4 +n_split=5 +layoutx=34 +layouty=24 +npx=1441 +npy=961 +io_layoutx=1 +io_layouty=1 +full_zs_filter=.true. +n_zs_filter=1 +n_del2_weak=24 +max_slope=0.15 +shal_cnv=.true. +do_deep=.true. + +npz=64 + +is_moving_nest=.false. + +quilting=.true. +write_dopost=.true. +write_tasks_per_group=120 +output_history=.true. + +output_grid=regional_latlon,regional_latlon +output_grid_cen_lon=180.0,{domlon} ;; central longitude +output_grid_cen_lat=0.0,{domlat} ;; central latitude +output_grid_lon_span=360.0,120.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=180.0,80.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.25,0.0625 ;; output grid spacing dlon . . . . +output_grid_dlat=0.25,0.0625 ;; output grid spacing dlat . . . . + +dt_atmos=120 + +# Thompson options +imp_physics=8 +iovr=3 +dt_inner=60 +dnats=0 +do_sat_adj=.false. +lgfdlmprad=.false. +nstf_n1=2 + +# Other RRFS physics options. +iaer=5111 +ltaerosol=.true. +satmedmf=.false. +do_mynnedmf=.true. +do_mynnsfclay=.true. +cdmbgwd=0.14,1.8,1.0,1.0 +iopt_sfc=4 +gwd_opt=2 +do_ugwp_v0=.true. +do_ugwp_v1=.false. +do_ugwp_v0_orog_only=.false. +do_ugwp_v0_nst_only=.false. +do_ugwp_v1_w_gsldrag=.false. +do_ugwp_v1_orog_only=.false. +do_gsl_drag_ls_bl=.false. +do_gsl_drag_ss=.false. +do_gsl_drag_tofd=.false. +bl_mynn_tkeadvect = .true. +lsm=2 + +# Diagnostics +lightning_threat=.true. + + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} +postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt +postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_840PE +ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES From 6f08940545ac2d817770cf556586a1a6abbeb9f7 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 5 Dec 2023 09:42:02 -0500 Subject: [PATCH 14/30] point to develop hafs gsi and develup ufs-community ufs-weather-model --- .gitmodules | 13 ++++--------- sorc/hafs_gsi.fd | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.gitmodules b/.gitmodules index 80bb1eea7..a464af520 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd - url = https://github.com/SamuelTrahanNOAA/ufs-weather-model.git - branch = WaterInTheWestC5-Nov2023 -# url = https://github.com/hafs-community/ufs-weather-model.git -# branch = support/hafs.v1.1.0 -# url = https://github.com/ufs-community/ufs-weather-model.git -# branch = develop + url = https://github.com/ufs-community/ufs-weather-model + branch = develop [submodule "UFS_UTILS"] path = sorc/hafs_utils.fd # url = https://github.com/hafs-community/UFS_UTILS.git @@ -19,9 +15,8 @@ branch = WaterInTheWestC5-Nov2023 [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd - url = https://github.com/SamuelTrahanNOAA/GSI.git -# branch = support/HAFS - branch = WaterInTheWestC5-Nov2023 + url = https://github.com/hafs-community/GSI.git + branch = support/HAFS [submodule "GPLOT"] path = sorc/hafs_graphics.fd/hrd_gplot url = https://github.com/hafs-community/GPLOT.git diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 0989b539a..8c86e8d41 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 0989b539aaf5f87e4c140e939c36cdeee08f543c +Subproject commit 8c86e8d418ad91ed25da7485f63704754ebaf77f From 0c9cdd657a067fca06fe935a2005f5df3bd7c803 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 5 Dec 2023 09:45:07 -0500 Subject: [PATCH 15/30] ufs-community ufs-weather-model develop for hafs_forecast.fd (for real this time) --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 1a4e3b2ac..f42e347b1 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 1a4e3b2aca9cb46c57ab29aad90081fb0dcc119f +Subproject commit f42e347b1cad15b4b2470fecb17cb19bb2d19459 From e580bca6862521145e3318103f8a881dd083ad38 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:53:35 -0500 Subject: [PATCH 16/30] patches and modules for c5 --- .../suite_FV3_global_nest_v0.xml | 95 +++++++++++++++++++ .../hafs_tracker.fd/gaea-c5-patch.diff | 78 +++++++++++++++ .../hafs_tracker.fd/gaea-c5.lua | 48 ++++++++++ .../hafs_utils.fd/build.gaea-c5.intel.lua | 54 +++++++++++ .../hafs_utils.fd/gaea-c5-patch.diff | 80 ++++++++++++++++ 5 files changed, 355 insertions(+) create mode 100644 sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml create mode 100644 sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff create mode 100644 sorc/customizations/hafs_tracker.fd/gaea-c5.lua create mode 100644 sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua create mode 100644 sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff diff --git a/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml b/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml new file mode 100644 index 000000000..36c2c2e41 --- /dev/null +++ b/sorc/customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml @@ -0,0 +1,95 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + sgscloud_radpre + GFS_rrtmg_pre + GFS_radiation_surface + rad_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + sgscloud_radpost + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + mynnsfc_wrapper + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + mynnedmf_wrapper + GFS_GWD_generic_pre + unified_ugwp + unified_ugwp_post + GFS_GWD_generic_post + GFS_suite_stateout_update + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + cu_c3_driver_pre + cu_c3_driver + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + mp_thompson_pre + + + mp_thompson + + + mp_thompson_post + GFS_MP_generic_post + cu_c3_driver_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + GFS_physics_post + + + + diff --git a/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff b/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff new file mode 100644 index 000000000..aa0e58437 --- /dev/null +++ b/sorc/customizations/hafs_tracker.fd/gaea-c5-patch.diff @@ -0,0 +1,78 @@ +diff --git a/src/build_all_cmake.sh b/src/build_all_cmake.sh +index 01a7bd7..818d59c 100755 +--- a/src/build_all_cmake.sh ++++ b/src/build_all_cmake.sh +@@ -1,8 +1,8 @@ + #!/bin/sh + +-set -eux ++# Do not set -eu on GAEA C5 or it'll fail. + source ./machine-setup.sh > /dev/null 2>&1 +- ++set -uex + + module use ../modulefiles + module load $target +@@ -24,7 +24,12 @@ elif [ $target = jet ]; then + elif [ $target = wcoss2 ] ; then + export FC=ftn + export F90=ftn ++ cmake_fortran_compiler=ifort + export CC=icc ++elif [[ $target =~ gaea ]] ; then ++ export FC=ftn ++ export F90=ftn ++ export CC=cc + else + echo "Unknown machine = $target" + exit 1 +@@ -36,7 +41,7 @@ if [ -d "build" ]; then + fi + mkdir build + cd build +-cmake .. -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_C_COMPILER=icc -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ++cmake .. -DCMAKE_Fortran_COMPILER=${cmake_fortran_compiler:-$FC} -DCMAKE_C_COMPILER=$CC -DCMAKE_BUILD_TYPE=${BUILD_TYPE} + make -j 8 VERBOSE=1 + make install + +diff --git a/src/machine-setup.sh b/src/machine-setup.sh +index 50dee4d..0138b80 100755 +--- a/src/machine-setup.sh ++++ b/src/machine-setup.sh +@@ -59,18 +59,25 @@ elif [[ -d /glade ]] ; then + target=yellowstone + module purge + elif [[ -d /lustre && -d /ncrc ]] ; then +- # We are on GAEA. +- if ( ! eval module help > /dev/null 2>&1 ) ; then +- # We cannot simply load the module command. The GAEA +- # /etc/profile modifies a number of module-related variables +- # before loading the module command. Without those variables, +- # the module command fails. Hence we actually have to source +- # /etc/profile here. +- echo load the module command 1>&2 +- source /etc/profile ++ if [[ $( hostname ) =~ gaea* ]] ; then ++ module purge ++ # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV ++ gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ ++ -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ ++ --pid=$$ --batch ++ ++ # Reload system default modules: ++ source /etc/bash.bashrc.local ++ ++ # Load EPIC's module directories: ++ source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh ++ fi ++ ++ if ( head /proc/cpuinfo | grep EPYC > /dev/null ) ; then ++ target=gaea-c5 ++ else ++ target=gaea + fi +- target=gaea +- module purge + elif [[ "$(hostname)" =~ "odin" ]]; then + target="odin" + else diff --git a/sorc/customizations/hafs_tracker.fd/gaea-c5.lua b/sorc/customizations/hafs_tracker.fd/gaea-c5.lua new file mode 100644 index 000000000..229e66fd2 --- /dev/null +++ b/sorc/customizations/hafs_tracker.fd/gaea-c5.lua @@ -0,0 +1,48 @@ +help([[ + This module loads libraries required for building and running UPP + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. +]]) + +whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) + +load("PrgEnv-intel/8.3.3") +load("intel-classic/2023.1.0") +load("cray-mpich/8.1.25") +load("python/3.9.12") + +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +load(pathJoin("stack-python", stack_python_ver)) + +local ufs_modules = { + {["jasper"] = "2.0.32"}, + {["zlib"] = "1.2.13"}, + {["libpng"] = "1.6.37"}, + {["hdf5"] = "1.14.0"}, + {["netcdf-c"] = "4.9.2"}, + {["netcdf-fortran"] = "4.6.0"}, + {["bacio"] = "2.4.1"}, + {["g2"] = "3.4.5"}, + {["g2tmpl"] = "1.10.2"}, + {["w3emc"] = "2.9.2"}, + {["w3nco"] = "2.4.1"}, + {["sigio"] = "2.3.2"}, +} + +for i = 1, #ufs_modules do + for name, default_version in pairs(ufs_modules[i]) do + local env_version_name = string.gsub(name, "-", "_") .. "_ver" + load(pathJoin(name, os.getenv(env_version_name) or default_version)) + end +end + +unload("darshan-runtime") +unload("cray-libsci") diff --git a/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua b/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua new file mode 100644 index 000000000..1e32a774f --- /dev/null +++ b/sorc/customizations/hafs_utils.fd/build.gaea-c5.intel.lua @@ -0,0 +1,54 @@ +help([[ + This module loads libraries required for building and running UPP + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. +]]) + +whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) + +load("PrgEnv-intel/8.3.3") +load("intel-classic/2023.1.0") +load("cray-mpich/8.1.25") +load("python/3.9.12") + +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +load(pathJoin("stack-python", stack_python_ver)) + +local ufs_modules = { + {["bacio"] = "2.4.1"}, + {["g2"] = "3.4.5"}, + {["ip"] = "3.3.3"}, + {["nemsio"] = "2.5.4"}, + {["sp"] = "2.3.3"}, + {["w3nco"] = "2.4.1"}, + {["sigio"] = "2.3.2"}, + {["sfcio"] = "1.4.1"}, + {["zlib"] = "1.2.13"}, + {["hdf5"] = "1.14.0"}, + {["netcdf-c"] = "4.9.2"}, + {["netcdf-fortran"] = "4.6.0"}, + {["esmf"] = "8.4.2"}, +} + +for i = 1, #ufs_modules do + for name, default_version in pairs(ufs_modules[i]) do + local env_version_name = string.gsub(name, "-", "_") .. "_ver" + load(pathJoin(name, os.getenv(env_version_name) or default_version)) + end +end + +unload("darshan-runtime") +unload("cray-libsci") + +setenv("CC","cc") +setenv("CXX","CC") +setenv("FC","ftn") +setenv("CMAKE_Platform","gaea_c5.intel") diff --git a/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff b/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff new file mode 100644 index 000000000..71ba8a662 --- /dev/null +++ b/sorc/customizations/hafs_utils.fd/gaea-c5-patch.diff @@ -0,0 +1,80 @@ +diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh +index 6e73630e..3478959c 100644 +--- a/sorc/machine-setup.sh ++++ b/sorc/machine-setup.sh +@@ -19,6 +19,14 @@ fi + target="" + USERNAME=`echo $LOGNAME | awk '{ print tolower($0)'}` + ++# Disable -e (abort on non-zero exit status) -u (abort on empty or ++# uninitialized variables) and -x (print every command executed) ++# because they can break system scripts. ++__ms_set_e=$( echo $- | grep e && echo YES || echo NO ) ++__ms_set_u=$( echo $- | grep u && echo YES || echo NO ) ++__ms_set_x=$( echo $- | grep x && echo YES || echo NO ) ++set +eux ++ + if [[ -d /lfs4 ]] ; then + # We are on NOAA Jet + if ( ! eval module help > /dev/null 2>&1 ) ; then +@@ -47,43 +55,25 @@ elif [[ -d /glade ]] ; then + target=cheyenne + module purge + elif [[ -d /lustre && -d /ncrc ]] ; then +- # We are on GAEA. +- if ( ! eval module help > /dev/null 2>&1 ) ; then +- # We cannot simply load the module command. The GAEA +- # /etc/profile modifies a number of module-related variables +- # before loading the module command. Without those variables, +- # the module command fails. Hence we actually have to source +- # /etc/profile here. +- source /etc/profile +- __ms_source_etc_profile=yes ++ if [[ $( hostname ) =~ gaea* ]] ; then ++ # Only runs on head nodes ++ module purge ++ # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV ++ gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ ++ -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ ++ --pid=$$ --batch ++ ++ # Reload system default modules: ++ source /etc/bash.bashrc.local ++ ++ # Load EPIC's module directories: ++ source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh ++ fi ++ if ( head /proc/cpuinfo | grep EPYC > /dev/null ) ; then ++ target=gaea-c5 + else +- __ms_source_etc_profile=no +- fi +- module purge > /dev/null 2>&1 +- module purge +-# clean up after purge +- unset _LMFILES_ +- unset _LMFILES_000 +- unset _LMFILES_001 +- unset LOADEDMODULES +- module load modules +- if [[ -d /opt/cray/ari/modulefiles ]] ; then +- module use -a /opt/cray/ari/modulefiles +- fi +- if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then +- module use -a /opt/cray/pe/ari/modulefiles +- fi +- if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then +- module use -a /opt/cray/pe/craype/default/modulefiles +- fi +- if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then +- source /etc/opt/cray/pe/admin-pe/site-config +- fi +- if [[ "$__ms_source_etc_profile" == yes ]] ; then +- source /etc/profile +- unset __ms_source_etc_profile ++ target=gaea + fi +- target=gaea + elif [[ "$(hostname)" =~ "Orion" ]]; then + target="orion" + module purge From 5c0524cc4ca6d4a59f7eb71a44fce62c8db1f5e9 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:54:07 -0500 Subject: [PATCH 17/30] global_nest_v0 suite --- sorc/build_forecast.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 6423debf5..cb767a2eb 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -5,6 +5,8 @@ if [ $target = wcoss2 ]; then source ../versions/build.ver; fi cwd=$(pwd) if [[ $target =~ .*c5 ]] ; then target=gaea-c5 ; fi +test -s hafs_forecast.fd/FV3/ccpp/suites/suite_FV3_global_nest_v0.xml || + ln -sf ../../../../customizations/hafs_forecast.fd/suite_FV3_global_nest_v0.xml hafs_forecast.fd/FV3/ccpp/suites/suite_FV3_global_nest_v0.xml cd hafs_forecast.fd/tests From b704f87467471ecc854c3ee30791dc273a5b87f8 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:54:26 -0500 Subject: [PATCH 18/30] apply patch to tracker for c5 support --- sorc/build_tracker.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sorc/build_tracker.sh b/sorc/build_tracker.sh index bb62b3da5..9438fd2ae 100755 --- a/sorc/build_tracker.sh +++ b/sorc/build_tracker.sh @@ -4,6 +4,13 @@ source ./machine-setup.sh > /dev/null 2>&1 if [ $target = wcoss2 ]; then source ../versions/build.ver; fi cwd=$(pwd) +if [[ ! -s hafs_tracker.fd/modulefiles/gaea-c5.lua ]] ; then + pushd hafs_tracker.fd + patch -p1 < ../customizations/hafs_tracker.fd/gaea-c5-patch.diff + popd + cp -fp customizations/hafs_tracker.fd/gaea-c5.lua hafs_tracker.fd/modulefiles/gaea-c5.lua +fi + #export BUILD_TYPE=DEBUG #export BUILD_TYPE=RELEASE export BUILD_TYPE=${BUILD_TYPE:-RELEASE} From 7273564a11c26bd7d0c043bbc6a6b2c6e79e3c76 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:54:47 -0500 Subject: [PATCH 19/30] apply patch to utils for c5 support --- sorc/build_utils.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sorc/build_utils.sh b/sorc/build_utils.sh index 5260a9032..bc2194b11 100755 --- a/sorc/build_utils.sh +++ b/sorc/build_utils.sh @@ -4,6 +4,13 @@ source ./machine-setup.sh > /dev/null 2>&1 if [ $target = wcoss2 ]; then source ../versions/build.ver; fi cwd=$(pwd) +if [[ ! -s hafs_utils.fd/modulefiles/build.gaea-c5.intel.lua ]] ; then + pushd hafs_utils.fd + patch -p1 < ../customizations/hafs_utils.fd/gaea-c5-patch.diff + popd + cp -fp customizations/hafs_utils.fd/build.gaea-c5.intel.lua hafs_utils.fd/modulefiles/build.gaea-c5.intel.lua +fi + cd hafs_utils.fd ./build_all.sh From b142e62add123edb41add9b7f7d203d317dfc6c8 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:55:12 -0500 Subject: [PATCH 20/30] do not compile gsi, hycom_utils, or ww3_utils --- sorc/hafs_build.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/hafs_build.cfg b/sorc/hafs_build.cfg index c7314ac7d..ba9c97475 100644 --- a/sorc/hafs_build.cfg +++ b/sorc/hafs_build.cfg @@ -7,9 +7,9 @@ Building tracker (tracker) ............................ yes Building utils (utils) ................................ yes Building tools (tools) ................................ yes - Building gsi (gsi) .................................... yes - Building hycom_utils (hycom_utils) .................... yes - Building ww3_utils (ww3_utils) ........................ yes + Building gsi (gsi) .................................... no + Building hycom_utils (hycom_utils) .................... no + Building ww3_utils (ww3_utils) ........................ no # -- END -- From 60e7fd97f5241e6e5f0653376a4362186aaad12a Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:55:38 -0500 Subject: [PATCH 21/30] longer wallclock limit for forecast so C768 can run --- rocoto/sites/gaea_c5.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/gaea_c5.ent b/rocoto/sites/gaea_c5.ent index 1724aee90..474d7f4c4 100644 --- a/rocoto/sites/gaea_c5.ent +++ b/rocoto/sites/gaea_c5.ent @@ -36,7 +36,7 @@ 1:ppn=8:tpp=1TOTAL_TASKS8NCTSK8OMP_THREADS100:10:00"> 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00"> - 02:59:00"> + 05:59:00"> OMP_THREADS2"> From 6c34811756653de4c0f66cee7e43a70e529f51b4 Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Tue, 12 Dec 2023 18:59:05 -0500 Subject: [PATCH 22/30] point to authoritative tracker and utils --- .gitmodules | 11 ++++------- sorc/hafs_tracker.fd | 2 +- sorc/hafs_utils.fd | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index a464af520..ecc4870d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,15 +4,12 @@ branch = develop [submodule "UFS_UTILS"] path = sorc/hafs_utils.fd -# url = https://github.com/hafs-community/UFS_UTILS.git -# branch = support/HAFS - url = https://github.com/SamuelTrahanNOAA/UFS_UTILS.git - branch = WaterInTheWestC5-Nov2023 + url = https://github.com/ufs-community/UFS_UTILS + branch = develop [submodule "gfdl-tracker"] path = sorc/hafs_tracker.fd - url = https://github.com/SamuelTrahanNOAA/gfdl-tracker.git -# branch = support/HAFS - branch = WaterInTheWestC5-Nov2023 + url = https://github.com/hafs-community/gfdl-tracker + branch = support/HAFS [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd url = https://github.com/hafs-community/GSI.git diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index fbb498c11..97a3b75dd 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit fbb498c1167c5dacd4d83a6be1ccfb61e5103f76 +Subproject commit 97a3b75dd40aef1b39b3cf436a8b2944082a87ae diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index 15ae4ccc3..340d87cec 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit 15ae4ccc3a4cd00aa1be4caa4b5d36dea35100e2 +Subproject commit 340d87cecd245ae2e97410e85a0860573e04a014 From aa19db812113bdf7596be0f019e85a511e9aff5e Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Mon, 18 Dec 2023 19:29:53 +0000 Subject: [PATCH 23/30] WaterInTheWest.sh: run Steve Case --- rocoto/WaterInTheWest.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh index f28182fd7..6d3f12760 100755 --- a/rocoto/WaterInTheWest.sh +++ b/rocoto/WaterInTheWest.sh @@ -10,9 +10,13 @@ EXPT=$(basename ${HOMEhafs}) SUBEXPT=$EXPT conf=../parm/WaterInTheWest.conf opts="-t -f" + +# To enable graphics, uncomment this. But only on Hera! +# conf="$conf config.run_pygraf=yes" + scrubopt="config.scrub_work=no config.scrub_com=no" # 3DEnVar with GDAS ensembles - ./run_hafs.py ${opts} 2020082506-2020082512 00L HISTORY \ + ./run_hafs.py ${opts} 2023022400 00L HISTORY \ config.EXPT=${EXPT} config.SUBEXPT=${SUBEXPT} \ - config.NHRS=12 ${scrubopt} \ + config.NHRS=24 ${scrubopt} \ $conf From d0cfcb3bce867e1f75e27e707f4401c82ed72a40 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Mon, 18 Dec 2023 19:30:18 +0000 Subject: [PATCH 24/30] Point to Sam's data, where the Steve Case resides. --- parm/system.conf.hera | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/system.conf.hera b/parm/system.conf.hera index 0f0c97e61..7e57b1f9f 100644 --- a/parm/system.conf.hera +++ b/parm/system.conf.hera @@ -24,12 +24,14 @@ COMIN={COMhafs} COMOUT={COMhafs} COMINnhc={ENV[DCOMROOT|-/dcom]}/nhc/atcf/ncep COMINjtwc={ENV[DCOMROOT|-/dcom]}/{ENV[PDY]}/wtxtbul/storm_data -COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16 +# COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16 +COMgfs=/scratch2/BMC/wrfruc/Samuel.Trahan/westwater/hafs-input/COMGFSv16 COMINobs={COMgfs} COMINgfs={COMgfs} COMINgdas={COMgfs} COMINarch={COMgfs}/syndat -COMrtofs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMRTOFSv2 +# COMrtofs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMRTOFSv2 +COMrtofs=/scratch2/BMC/wrfruc/Samuel.Trahan/westwater/hafs-input/COMRTOFSv2 COMINrtofs={COMrtofs} COMINmsg={COMINgfs} COMINhafs={COMINgfs} From d7f8139a6ac65388706422a653a141503f3d2405 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Mon, 18 Dec 2023 19:42:14 +0000 Subject: [PATCH 25/30] point to NOAA-GSL for pygraf --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ecc4870d8..719829ac3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -24,5 +24,5 @@ branch = support/HAFS [submodule "sorc/hafs_graphics.fd/pygraf"] path = sorc/hafs_graphics.fd/pygraf - url = https://github.com/SamuelTrahanNOAA/pygraf - branch = hafs-ar + url = https://github.com/NOAA-GSL/pygraf + branch = arfs_dev From 3d4552bf31d92b489dc656bfebe600ded55a1566 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Mon, 18 Dec 2023 19:43:27 +0000 Subject: [PATCH 26/30] revert changes to gsi submodule --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 719829ac3..03b471991 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,7 +13,8 @@ [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd url = https://github.com/hafs-community/GSI.git - branch = support/HAFS +# branch = support/HAFS + branch = support/hafs.v1.1.0 [submodule "GPLOT"] path = sorc/hafs_graphics.fd/hrd_gplot url = https://github.com/hafs-community/GPLOT.git From 2b0897ae3d36e7010a0d4333d0b19d17a895c2d0 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Tue, 19 Dec 2023 05:18:44 +0000 Subject: [PATCH 27/30] correct non-nested global parm file descriptions --- parm/WaterInTheWestC384NoNest.conf | 2 +- parm/WaterInTheWestC384Uniform.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/WaterInTheWestC384NoNest.conf b/parm/WaterInTheWestC384NoNest.conf index ffa22dbd1..1441c654c 100644 --- a/parm/WaterInTheWestC384NoNest.conf +++ b/parm/WaterInTheWestC384NoNest.conf @@ -1,5 +1,5 @@ # This is a UNIX conf file that contains ALL information relating to -# the HAFS basin-focused global-nesting configuration. +# the HAFS stretched globe configuration with no nest. # [config] # Specify the domain center Latitude and Longitude diff --git a/parm/WaterInTheWestC384Uniform.conf b/parm/WaterInTheWestC384Uniform.conf index 0eb365aaf..f7421e4ae 100644 --- a/parm/WaterInTheWestC384Uniform.conf +++ b/parm/WaterInTheWestC384Uniform.conf @@ -1,5 +1,5 @@ # This is a UNIX conf file that contains ALL information relating to -# the HAFS basin-focused global-nesting configuration. +# the HAFS uniform C384 global configuration (with no nest) # [config] # Specify the domain center Latitude and Longitude From 27704ab615d91b0ded4b8c97e97a2d7238cd3f1e Mon Sep 17 00:00:00 2001 From: Samuel Trahan Date: Wed, 20 Dec 2023 16:09:22 -0500 Subject: [PATCH 28/30] disable pygraf on GAEA (untested on Hera!!) --- parm/hafs.conf | 2 ++ parm/system.conf.gaea | 2 ++ rocoto/WaterInTheWest.sh | 6 +++--- sorc/hafs_graphics.fd/pygraf | 2 +- ush/hafs/launcher.py | 4 ++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index c5bd548b8..355996c63 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -30,6 +30,8 @@ input_catalog=fcst_{GFSVER} docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, or NONE) datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE) +pygraf_can_run=yes + ## Configure file and directory paths [dir] HOMEhafs={CDSAVE}/{EXPT} diff --git a/parm/system.conf.gaea b/parm/system.conf.gaea index f3592c912..bb289679d 100644 --- a/parm/system.conf.gaea +++ b/parm/system.conf.gaea @@ -9,6 +9,8 @@ cpu_account=gsl-glo ## Archive path archive=hpss:/BMC/{tape_project}/2year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar +pygraf_can_run=no + [dir] gaea_input=/lustre/f2/dev/Samuel.Trahan/hafs/hafs-input hafs_work=/lustre/f2/dev/{ENV[USER]}/hafs/work diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh index 6d3f12760..f537502cb 100755 --- a/rocoto/WaterInTheWest.sh +++ b/rocoto/WaterInTheWest.sh @@ -7,8 +7,8 @@ set -x cd ${HOMEhafs}/rocoto EXPT=$(basename ${HOMEhafs}) -SUBEXPT=$EXPT -conf=../parm/WaterInTheWest.conf +SUBEXPT=pygrafC192no +conf=../parm/WaterInTheWestC192.conf opts="-t -f" # To enable graphics, uncomment this. But only on Hera! @@ -19,4 +19,4 @@ scrubopt="config.scrub_work=no config.scrub_com=no" ./run_hafs.py ${opts} 2023022400 00L HISTORY \ config.EXPT=${EXPT} config.SUBEXPT=${SUBEXPT} \ config.NHRS=24 ${scrubopt} \ - $conf + $conf config.run_pygraf=yes diff --git a/sorc/hafs_graphics.fd/pygraf b/sorc/hafs_graphics.fd/pygraf index 84e411224..9003a692b 160000 --- a/sorc/hafs_graphics.fd/pygraf +++ b/sorc/hafs_graphics.fd/pygraf @@ -1 +1 @@ -Subproject commit 84e4112245e10df1d95b7f2dd6426673b71f913f +Subproject commit 9003a692bc94b3561f78844585ad9165b2d6bfea diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 14438bc25..ea58f828d 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -578,6 +578,10 @@ def launch(file_list,cycle,stid,moreopt,case_root,init_dirs=True, revit=conf.read_fake_tcvitals() conf.gen_vitals(stid,cycling_interval,revit) + if conf.getbool('config', 'run_pygraf') and not conf.getbool('config','pygraf_can_run'): + logger.warning("Will disable pygraf because it cannot run on this system (config.pygraf_can_run=no).") + conf.set('config', 'run_pygraf', 'no') + # rocoto does not initialize the dirs, it returns here. if not init_dirs: if prelaunch is not None: From 025d823829d2c5e9bdb2b936767de76cd68ed7b3 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Thu, 21 Dec 2023 00:33:24 +0000 Subject: [PATCH 29/30] revert changes to WaterInTheWest.sh --- rocoto/WaterInTheWest.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rocoto/WaterInTheWest.sh b/rocoto/WaterInTheWest.sh index f537502cb..93ba7d5f5 100755 --- a/rocoto/WaterInTheWest.sh +++ b/rocoto/WaterInTheWest.sh @@ -7,13 +7,10 @@ set -x cd ${HOMEhafs}/rocoto EXPT=$(basename ${HOMEhafs}) -SUBEXPT=pygrafC192no -conf=../parm/WaterInTheWestC192.conf +SUBEXPT=$EXPT +conf=../parm/WaterInTheWest.conf opts="-t -f" -# To enable graphics, uncomment this. But only on Hera! -# conf="$conf config.run_pygraf=yes" - scrubopt="config.scrub_work=no config.scrub_com=no" # 3DEnVar with GDAS ensembles ./run_hafs.py ${opts} 2023022400 00L HISTORY \ From 63e72b277aa415fe1033362878b038869c6dd7c6 Mon Sep 17 00:00:00 2001 From: "samuel.trahan" Date: Thu, 21 Dec 2023 00:34:18 +0000 Subject: [PATCH 30/30] reduce the NoNest and Uniform conf files to override values --- parm/WaterInTheWestC384NoNest.conf | 165 +-------------------------- parm/WaterInTheWestC384Uniform.conf | 166 +--------------------------- 2 files changed, 4 insertions(+), 327 deletions(-) diff --git a/parm/WaterInTheWestC384NoNest.conf b/parm/WaterInTheWestC384NoNest.conf index 1441c654c..289bbb00e 100644 --- a/parm/WaterInTheWestC384NoNest.conf +++ b/parm/WaterInTheWestC384NoNest.conf @@ -1,176 +1,15 @@ -# This is a UNIX conf file that contains ALL information relating to +# This is a UNIX conf file that overrrides WaterInTheWestC384.conf to run # the HAFS stretched globe configuration with no nest. +# This file must be after that one in the run_hafs.py command line. # -[config] -# Specify the domain center Latitude and Longitude -domlat=32.5 -domlon=-135.0 -# Same as domlat and domlon but vitals formatted -vitalsdomlat=325N -vitalsdomlon=1350W - -# HAFS launcher requires vitals and a storm. This is a placeholder for the global static nest. -fakestormid=00L -fakestormname=NATL -fakestormorg=NHC -fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M - -run_atm_mvnest=no -run_wave=no -ww3_model=ww3 -run_ocean=no -ocean_model=hycom - -run_atm_init=no -run_atm_init_fgat=no -run_atm_init_ens=no -run_atm_merge=no -run_atm_merge_fgat=no -run_atm_merge_ens=no -run_atm_vi=no -run_atm_vi_fgat=no -run_atm_vi_ens=no -run_gsi=no -gsi_d01=no -gsi_d02=no -run_analysis_merge=no -run_analysis_merge_ens=no -run_fgat=no -run_envar=no -run_ensda=no -run_enkf=no - -[dir] -PARMforecast={PARMhafs}/forecast/globnest [grid] -CASE=C384 ;; FV3 resolution -LEVS=65 ;; Model vertical levels: 65 gtype=stretch ;; grid type: uniform, stretch, nest, or regional -# If gridfixdir is provided and the dir exists, then use the pre-generated static grid fix files under gridfixdir -#gridfixdir=/let/hafs_grid/generate/grid -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/HAFS-weird-C768 -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest -# Otherwise, will generate the model grid according to the following grid parameters -# Need for grid types: stretch, nest and regional -stretch_fac=1.0001 ;; Stretching factor for the grid -target_lon={domlon} ;; center longitude of the highest resolution tile -target_lat={domlat} ;; center latitude of the highest resolution tile -# Need for grid types: nest and regional nest_grids=0 -parent_grid_num=1 -parent_tile=6 -refine_ratio=4 -istart_nest=25 -jstart_nest=73 -iend_nest=744 -jend_nest=648 -regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v0 -ccpp_suite_glob=FV3_global_nest_v0 -ccpp_suite_nest=FV3_global_nest_v0 - -# For the global domain if it is a global or global-nesting experiment -glob_k_split=1 -glob_n_split=7 glob_layoutx=10 glob_layouty=10 -glob_npx=385 -glob_npy=385 -glob_io_layoutx=1 -glob_io_layouty=1 -glob_full_zs_filter=.true. -glob_n_zs_filter=0 -glob_n_del2_weak=15 -glob_max_slope=0.15 -glob_shal_cnv=.true. -glob_do_deep=.true. -glob_imfshalcnv=5 -glob_imfdeepcnv=5 -# For the nested or regional standalone domain -k_split=4 -n_split=5 -layoutx=34 -layouty=24 -npx=1441 -npy=961 -io_layoutx=1 -io_layouty=1 -full_zs_filter=.true. -n_zs_filter=1 -n_del2_weak=24 -max_slope=0.15 -shal_cnv=.true. -do_deep=.true. - -npz=64 - -is_moving_nest=.false. - -quilting=.true. -write_dopost=.true. -write_tasks_per_group=120 -output_history=.true. - -output_grid=regional_latlon,regional_latlon -output_grid_cen_lon=180.0,{domlon} ;; central longitude -output_grid_cen_lat=0.0,{domlat} ;; central latitude -output_grid_lon_span=360.0,120.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) -output_grid_lat_span=180.0,80.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) -output_grid_dlon=0.25,0.0625 ;; output grid spacing dlon . . . . -output_grid_dlat=0.25,0.0625 ;; output grid spacing dlat . . . . - -dt_atmos=120 - -# Thompson options -imp_physics=8 -iovr=3 -dt_inner=60 -dnats=0 -do_sat_adj=.false. -lgfdlmprad=.false. -nstf_n1=2 - -# Other RRFS physics options. -iaer=5111 -ltaerosol=.true. -satmedmf=.false. -do_mynnedmf=.true. -do_mynnsfclay=.true. -cdmbgwd=0.14,1.8,1.0,1.0 -iopt_sfc=4 -gwd_opt=2 -do_ugwp_v0=.true. -do_ugwp_v1=.false. -do_ugwp_v0_orog_only=.false. -do_ugwp_v0_nst_only=.false. -do_ugwp_v1_w_gsldrag=.false. -do_ugwp_v1_orog_only=.false. -do_gsl_drag_ls_bl=.false. -do_gsl_drag_ss=.false. -do_gsl_drag_tofd=.false. -bl_mynn_tkeadvect = .true. -lsm=2 - -# Diagnostics -lightning_threat=.true. - - -[atm_post] -satpost=.false. -post_gridspecs=auto,auto -trak_gridspecs={post_gridspecs} -postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt -postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt - -[atm_post_ens] -satpost_ens=.false. -post_gridspecs_ens=auto,auto -trak_gridspecs_ens={post_gridspecs} [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_840PE -ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES diff --git a/parm/WaterInTheWestC384Uniform.conf b/parm/WaterInTheWestC384Uniform.conf index f7421e4ae..6076b8d8e 100644 --- a/parm/WaterInTheWestC384Uniform.conf +++ b/parm/WaterInTheWestC384Uniform.conf @@ -1,176 +1,14 @@ -# This is a UNIX conf file that contains ALL information relating to +# This is a UNIX conf file that overrrides WaterInTheWestC384.conf to run # the HAFS uniform C384 global configuration (with no nest) +# This file must be after that one in the run_hafs.py command line. # -[config] -# Specify the domain center Latitude and Longitude -domlat=32.5 -domlon=-135.0 -# Same as domlat and domlon but vitals formatted -vitalsdomlat=325N -vitalsdomlon=1350W - -# HAFS launcher requires vitals and a storm. This is a placeholder for the global static nest. -fakestormid=00L -fakestormname=NATL -fakestormorg=NHC -fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M - -run_atm_mvnest=no -run_wave=no -ww3_model=ww3 -run_ocean=no -ocean_model=hycom - -run_atm_init=no -run_atm_init_fgat=no -run_atm_init_ens=no -run_atm_merge=no -run_atm_merge_fgat=no -run_atm_merge_ens=no -run_atm_vi=no -run_atm_vi_fgat=no -run_atm_vi_ens=no -run_gsi=no -gsi_d01=no -gsi_d02=no -run_analysis_merge=no -run_analysis_merge_ens=no -run_fgat=no -run_envar=no -run_ensda=no -run_enkf=no - -[dir] -PARMforecast={PARMhafs}/forecast/globnest - [grid] -CASE=C384 ;; FV3 resolution -LEVS=65 ;; Model vertical levels: 65 gtype=uniform ;; grid type: uniform, stretch, nest, or regional -# If gridfixdir is provided and the dir exists, then use the pre-generated static grid fix files under gridfixdir -#gridfixdir=/let/hafs_grid/generate/grid -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/HAFS-weird-C768 -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest -#gridfixdir=/scratch1/BMC/zrtrr/Samuel.Trahan/westwater/C768_nest -# Otherwise, will generate the model grid according to the following grid parameters -# Need for grid types: stretch, nest and regional -stretch_fac=1.0001 ;; Stretching factor for the grid -target_lon={domlon} ;; center longitude of the highest resolution tile -target_lat={domlat} ;; center latitude of the highest resolution tile -# Need for grid types: nest and regional nest_grids=0 -parent_grid_num=1 -parent_tile=6 -refine_ratio=4 -istart_nest=25 -jstart_nest=73 -iend_nest=744 -jend_nest=648 -regional_esg=no [forecast] -ccpp_suite_regional=FV3_global_nest_v0 -ccpp_suite_glob=FV3_global_nest_v0 -ccpp_suite_nest=FV3_global_nest_v0 - -# For the global domain if it is a global or global-nesting experiment -glob_k_split=1 -glob_n_split=7 glob_layoutx=10 glob_layouty=10 -glob_npx=385 -glob_npy=385 -glob_io_layoutx=1 -glob_io_layouty=1 -glob_full_zs_filter=.true. -glob_n_zs_filter=0 -glob_n_del2_weak=15 -glob_max_slope=0.15 -glob_shal_cnv=.true. -glob_do_deep=.true. -glob_imfshalcnv=5 -glob_imfdeepcnv=5 -# For the nested or regional standalone domain -k_split=4 -n_split=5 -layoutx=34 -layouty=24 -npx=1441 -npy=961 -io_layoutx=1 -io_layouty=1 -full_zs_filter=.true. -n_zs_filter=1 -n_del2_weak=24 -max_slope=0.15 -shal_cnv=.true. -do_deep=.true. - -npz=64 - -is_moving_nest=.false. - -quilting=.true. -write_dopost=.true. -write_tasks_per_group=120 -output_history=.true. - -output_grid=regional_latlon,regional_latlon -output_grid_cen_lon=180.0,{domlon} ;; central longitude -output_grid_cen_lat=0.0,{domlat} ;; central latitude -output_grid_lon_span=360.0,120.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) -output_grid_lat_span=180.0,80.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) -output_grid_dlon=0.25,0.0625 ;; output grid spacing dlon . . . . -output_grid_dlat=0.25,0.0625 ;; output grid spacing dlat . . . . - -dt_atmos=120 - -# Thompson options -imp_physics=8 -iovr=3 -dt_inner=60 -dnats=0 -do_sat_adj=.false. -lgfdlmprad=.false. -nstf_n1=2 - -# Other RRFS physics options. -iaer=5111 -ltaerosol=.true. -satmedmf=.false. -do_mynnedmf=.true. -do_mynnsfclay=.true. -cdmbgwd=0.14,1.8,1.0,1.0 -iopt_sfc=4 -gwd_opt=2 -do_ugwp_v0=.true. -do_ugwp_v1=.false. -do_ugwp_v0_orog_only=.false. -do_ugwp_v0_nst_only=.false. -do_ugwp_v1_w_gsldrag=.false. -do_ugwp_v1_orog_only=.false. -do_gsl_drag_ls_bl=.false. -do_gsl_drag_ss=.false. -do_gsl_drag_tofd=.false. -bl_mynn_tkeadvect = .true. -lsm=2 - -# Diagnostics -lightning_threat=.true. - - -[atm_post] -satpost=.false. -post_gridspecs=auto,auto -trak_gridspecs={post_gridspecs} -postxconfig_satpost={PARMhafs}/post/postxconfig-NT-AR-HAFS-SAT.txt -postxconfig_nosat={PARMhafs}/post/postxconfig-NT-AR-HAFS-PRSLEV.txt - -[atm_post_ens] -satpost_ens=.false. -post_gridspecs_ens=auto,auto -trak_gridspecs_ens={post_gridspecs} [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_840PE -ATM_POST_RESOURCES=ATM_POST_INLINE_RESOURCES