From 543a94e1a429911c29c4dd704ef51c21b552bea3 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Tue, 3 May 2022 13:53:10 -0600 Subject: [PATCH] Standardize static data across Tier-1 platforms; fix and improve IC and LBC data retrieval (#744) --- scripts/exregional_get_extrn_mdl_files.sh | 4 +- tests/WE2E/run_WE2E_tests.sh | 22 +++++----- ...NUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh | 1 + ...mpact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...pact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh | 3 ++ ...km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh | 2 + ...mpact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...5km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...t_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 2 + ...ompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 3 ++ ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 2 + ...mpact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 2 + ...ig.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh | 2 + .../wflow_features/config.subhourly_post.sh | 2 + .../config.subhourly_post_ensemble_2mems.sh | 2 + ush/NOMADS_get_extrn_mdl_files.sh | 12 +++--- ush/config_defaults.sh | 41 ++++++++++++------- ush/machine/cheyenne.sh | 25 +++++------ ush/machine/gaea.sh | 14 +++---- ush/machine/hera.sh | 27 ++++++------ ush/machine/jet.sh | 21 ++++++---- ush/machine/noaacloud.sh | 33 ++++++++------- ush/machine/odin.sh | 32 +++++++++++---- ush/machine/orion.sh | 27 ++++++------ ush/machine/singularity.sh | 4 +- ush/machine/stampede.sh | 34 +++++++++++---- ush/machine/wcoss_dell_p3.sh | 27 ++++++------ ush/retrieve_data.py | 2 +- ush/set_extrn_mdl_params.sh | 17 -------- ush/setup.sh | 23 ++++++----- ush/templates/data_locations.yml | 2 - 37 files changed, 242 insertions(+), 164 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index fc02e3b996..5a3fe04cf3 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -108,7 +108,7 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi -data_stores="hpss aws" +data_stores="${EXTRN_MDL_DATA_STORES}" yyyymmddhh=${extrn_mdl_cdate:0:10} yyyy=${yyyymmddhh:0:4} @@ -141,7 +141,7 @@ if [ -n "${file_names:-}" ] ; then fi if [ -n "${input_file_path:-}" ] ; then - data_stores="disk hpss aws" + data_stores="disk $data_stores" additional_flags="$additional_flags \ --input_file_path ${input_file_path}" fi diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 330669c5ee..a11dfaba3a 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -995,6 +995,9 @@ PTMP=\"${PTMP}\"" # if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then + # Ensure we only check on disk for these files + data_stores="disk" + extrn_mdl_source_basedir=${TEST_EXTRN_MDL_SOURCE_BASEDIR:-} if [ ! -d "${extrn_mdl_source_basedir:-}" ] ; then print_err_msg_exit "\ @@ -1005,20 +1008,16 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_ICS" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${yyyymmddhh}" else - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${yyyymmddhh}" fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" - elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${yyyymmddhh}" else - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${yyyymmddhh}" fi # # Make sure that the forecast length is evenly divisible by the interval @@ -1039,10 +1038,11 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS} +EXTRN_MDL_SOURCE_BASEDIR_ICS='${EXTRN_MDL_SOURCE_BASEDIR_ICS}' EXTRN_MDL_FILES_ICS=( ${EXTRN_MDL_FILES_ICS[@]} ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS} -EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} )" +EXTRN_MDL_SOURCE_BASEDIR_LBCS='${EXTRN_MDL_SOURCE_BASEDIR_LBCS}' +EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} ) +EXTRN_MDL_DATA_STORES=\"$data_stores\"" fi # diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh index 492a163360..1465641a59 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh @@ -22,6 +22,7 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 4239752de4..40f5e4997e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 8aaeddf43c..f44afffcda 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 9e86198e6f..afcfa32e16 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh index 67b381044c..f1302d1634 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" @@ -23,3 +25,4 @@ CYCL_HRS=( "00" ) FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="3" + diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh index 09b9548d11..0060e44663 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index f4b781e070..3dfedb5686 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index b0b8a2e42d..bf2e2f15e5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 2a98569a39..8fc60571db 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 2407157518..11227ea005 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index eda52d1e7e..396ce3e150 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) + DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 1b3131663d..cc92aecaaa 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index cf6965435f..a75f8d79e6 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 3534d5df9e..c5512d0ead 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index c2a054b309..e5c4e7732d 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh index 5d6ed126ec..61d4e5c9eb 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index d985559f2e..7a10a0ec34 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh index 09df72e29a..a1676e4d58 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh @@ -15,6 +15,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh index 4c20807119..adcc37a332 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh @@ -21,6 +21,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/ush/NOMADS_get_extrn_mdl_files.sh b/ush/NOMADS_get_extrn_mdl_files.sh index 308aa80827..789eeff3dc 100755 --- a/ush/NOMADS_get_extrn_mdl_files.sh +++ b/ush/NOMADS_get_extrn_mdl_files.sh @@ -37,10 +37,10 @@ cd gfs.$yyyymmdd/$hh #getting online analysis data if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f000 + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f000 else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmanl.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcanl.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmanl.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcanl.nemsio fi #getting online forecast data @@ -60,10 +60,10 @@ echo $ifcst echo $ifcst_str # if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f${ifcst_str} + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f${ifcst_str} else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmf${ifcst_str}.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcf${ifcst_str}.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmf${ifcst_str}.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcf${ifcst_str}.nemsio fi # ifcst=$[$ifcst+$nfcst_int] diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index aa6cddc368..8af4590a6e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -238,19 +238,6 @@ EXEC_SUBDIR="bin" # need in order to create initial and boundary condition files for a given # cycle on the native FV3-LAM grid. # -# FIXLAM_NCO_BASEDIR: -# The base directory containing pregenerated grid, orography, and surface -# climatology files. For the pregenerated grid specified by PREDEF_GRID_NAME, -# these "fixed" files are located in: -# -# ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME} -# -# The workflow scripts will create a symlink in the experiment directory -# that will point to a subdirectory (having the name of the grid being -# used) under this directory. This variable should be set to a null -# string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN). -# # envir, NET, model_ver, RUN: # Standard environment variables defined in the NCEP Central Operations WCOSS # Implementation Standards document as follows: @@ -292,7 +279,7 @@ EXEC_SUBDIR="bin" #----------------------------------------------------------------------- # COMIN="/path/of/directory/containing/data/files/for/IC/LBCS" -FIXLAM_NCO_BASEDIR="" +STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" envir="para" NET="rrfs" model_ver="v1.0.0" @@ -711,6 +698,13 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # EXTRN_MDL_FILES_LBCS: # Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. # +# EXTRN_MDL_DATA_STORES: +# A list of data stores where the scripts should look for external model +# data. The list is in priority order. If disk information is provided +# via USE_USER_STAGED_EXTRN_FILES or a known location on the platform, +# the disk location will be highest priority. Options are disk, hpss, +# aws, and nomads. +# #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES="FALSE" @@ -718,6 +712,7 @@ EXTRN_MDL_SOURCE_BASEDIR_ICS="" EXTRN_MDL_FILES_ICS="" EXTRN_MDL_SOURCE_BASEDIR_LBCS="" EXTRN_MDL_FILES_LBCS="" +EXTRN_MDL_DATA_STORES="" # #----------------------------------------------------------------------- # @@ -1305,6 +1300,22 @@ VX_ENSPOINT_PROB_TN="run_enspointvx_prob" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # +# DOMAIN_PREGEN_BASEDIR: +# The base directory containing pregenerated grid, orography, and surface +# climatology files. This is an alternative for setting GRID_DIR, +# OROG_DIR, and SFC_CLIMO_DIR individually +# +# For the pregenerated grid specified by PREDEF_GRID_NAME, +# these "fixed" files are located in: +# +# ${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME} +# +# The workflow scripts will create a symlink in the experiment directory +# that will point to a subdirectory (having the name of the grid being +# used) under this directory. This variable should be set to a null +# string in this file, but it can be specified in the user-specified +# workflow configuration file (EXPT_CONFIG_FN). +# # RUN_TASK_GET_EXTRN_ICS: # Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. # @@ -1351,6 +1362,8 @@ OROG_DIR="/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO="TRUE" SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files" +DOMAIN_PREGEN_BASEDIR="" + RUN_TASK_GET_EXTRN_ICS="TRUE" RUN_TASK_GET_EXTRN_LBCS="TRUE" RUN_TASK_MAKE_ICS="TRUE" diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index f92c6fa844..cf80db74f4 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -46,12 +46,13 @@ QUEUE_HPSS=${QUEUE_HPSS:-"regular"} QUEUE_FCST=${QUEUE_FCST:-"regular"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} -FIXaer=${FIXaer:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_aer"} -FIXlut=${FIXlut:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +staged_data_dir="/glade/p/ral/jntp/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -62,12 +63,12 @@ RUN_CMD_POST='mpirun -np $nprocs' # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/glade/p/ral/jntp/UFS_SRW_app/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 9bad0af743..37f0838e6f 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -48,11 +48,13 @@ QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} WTIME_MAKE_LBCS="00:60:00" # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"//lustre/f2/dev/Mark.Potts/EPIC/fix/fix_am"} -FIXaer=${FIXaer:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_aer"} -FIXlut=${FIXlut:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_sfc_climo"} +staged_data_dir="/lustre/f2/dev/Mark.Potts/EPIC/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" RUN_CMD_SERIAL="time" #Run Commands currently differ for GNU/openmpi @@ -66,5 +68,3 @@ RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations # MET Plus is not yet supported on gaea # Test Data Locations -TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" - diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index 39c28f8ee4..454a1e9383 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -28,6 +28,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -49,12 +51,13 @@ PARTITION_FCST=${PARTITION_FCST:-"hera"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} -FIXaer=${FIXaer:-"/scratch1/NCEPDEV/global/glopara/fix/fix_aer"} -FIXlut=${FIXlut:-"/scratch1/NCEPDEV/global/glopara/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} +staged_data_dir="/scratch2/BMC/det/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -65,14 +68,14 @@ RUN_CMD_POST="srun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/develop/model_data" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index 9a12c0c142..f383090ec5 100644 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -44,6 +44,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -65,12 +67,13 @@ PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} -FIXaer=${FIXaer:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_aer"} -FIXlut=${FIXlut:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen"} +staged_data_dir="/mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -79,6 +82,6 @@ RUN_CMD_FCST="srun" RUN_CMD_POST="srun" # Test Data Locations -TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/staged_extrn_mdl_files" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/noaacloud.sh b/ush/machine/noaacloud.sh index fe42766612..49299c325f 100755 --- a/ush/machine/noaacloud.sh +++ b/ush/machine/noaacloud.sh @@ -1,6 +1,5 @@ -#!/bin/bash +#!/bin/bash -set -x function file_location() { @@ -16,16 +15,13 @@ function file_location() { "FV3GFS") location='/contrib/GST/model_data/FV3GFS/${yyyymmdd}${hh}' ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Hera. Will try to pull from HPSS" - ;; esac echo ${location:-} } +export OPT=/contrib/EPIC/hpc-modules +export PATH=${PATH}:/contrib/GST/miniconda/envs/regional_workflow/bin EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ @@ -34,6 +30,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -50,22 +48,23 @@ NCORES_PER_NODE=${NCORES_PER_NODE:-36} SCHED=${SCHED:-"slurm"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/contrib/EPIC/fix/fix_am"} -FIXaer=${FIXaer:-"/contrib/EPIC/fix/fix_aer"} -FIXlut=${FIXlut:-"/contrib/EPIC/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/contrib/EPIC/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/EPIC/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} +staged_data_dir="/contrib/EPIC/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" RUN_CMD_SERIAL="time" #Run Commands currently differ for GNU/openmpi #RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' #RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' #RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' -RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' -RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' -RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' +RUN_CMD_UTILS='mpiexec -np $nprocs' +RUN_CMD_FCST='mpiexec -np ${PE_MEMBER01}' +RUN_CMD_POST='mpiexec -np $nprocs' # MET Installation Locations # MET Plus is not yet supported on noaacloud - +. /contrib/EPIC/.bash_conda diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh index 1bceaa873e..58d360a633 100644 --- a/ush/machine/odin.sh +++ b/ush/machine/odin.sh @@ -9,16 +9,26 @@ function file_location() { external_model=${1} external_file_fmt=${2} + staged_data_dir="/scratch/ywang/UFS_SRW_App/develop" + location="" case ${external_model} in "GSMGFS") - location='/scratch/ywang/EPIC/GDAS/2019053000_mem001' + location="${staged_data_dir}/input_model_data/GFS" ;; "FV3GFS") - location='/scratch/ywang/test_runs/FV3_regional/gfs/${yyyymmdd}' + location="${staged_data_dir}/input_model_data/FV3GFS" + ;; + "HRRR") + location="${staged_data_dir}/input_model_data/HRRR" + ;; + "RAP") + location="${staged_data_dir}/input_model_data/RAP" + ;; + "NAM") + location="${staged_data_dir}/input_model_data/NAM" ;; - esac echo ${location:-} @@ -52,15 +62,19 @@ PARTITION_FCST=${PARTITION_FCST:-"workq"} QUEUE_FCST=${QUEUE_FCST:-"workq"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} -FIXaer=${FIXaer:-"/scratch/ywang/fix/theia_fix/fix_aer"} -FIXlut=${FIXlut:-"/scratch/ywang/fix/theia_fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="srun -n 1" RUN_CMD_UTILS='srun -n $nprocs' RUN_CMD_FCST='srun -n ${PE_MEMBER01}' RUN_CMD_POST="srun -n 1" + +# Test Data Locations +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index ce39308d4e..b8032e2412 100644 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -22,6 +22,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -43,12 +45,13 @@ PARTITION_FCST=${PARTITION_FCST:-"orion"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} -FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} -FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/work/noaa/fv3-cam/UFS_SRW_App/FIXLAM_NCO_BASE"} +staged_data_dir="/work/noaa/fv3-cam/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -59,13 +62,13 @@ RUN_CMD_POST="srun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.0"} METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/FV3LAM_pregen" -TEST_COMIN="/work/noaa/fv3-cam/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/develop/model_data" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/singularity.sh b/ush/machine/singularity.sh index 528e1dbd5c..14f840800b 100644 --- a/ush/machine/singularity.sh +++ b/ush/machine/singularity.sh @@ -21,6 +21,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -47,7 +49,7 @@ FIXaer=${FIXaer:-"/contrib/global/glopara/fix/fix_aer"} FIXlut=${FIXlut:-"/contrib/global/glopara/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/contrib/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"/needs/to/be/specified"} # Run commands for executables RUN_CMD_SERIAL="time" diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh index 41afa5fc17..3f879ea22f 100644 --- a/ush/machine/stampede.sh +++ b/ush/machine/stampede.sh @@ -9,15 +9,26 @@ function file_location() { external_model=${1} external_file_fmt=${2} + staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" + location="" case ${external_model} in "GSMGFS") - ;& # Fall through. All files in same place + location="${staged_data_dir}/input_model_data/GFS" + ;; "FV3GFS") - location='/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001' + location="${staged_data_dir}/input_model_data/FV3GFS" + ;; + "HRRR") + location="${staged_data_dir}/input_model_data/HRRR" + ;; + "RAP") + location="${staged_data_dir}/input_model_data/RAP" + ;; + "NAM") + location="${staged_data_dir}/input_model_data/NAM" ;; - esac echo ${location:-} @@ -51,15 +62,20 @@ PARTITION_FCST=${PARTITION_FCST:-"normal"} QUEUE_FCST=${QUEUE_FCST:-"normal"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} -FIXaer=${FIXaer:-"/work/00315/tg455890/stampede2/regional_fv3/fix_aer"} -FIXlut=${FIXlut:-"/work/00315/tg455890/stampede2/regional_fv3/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS='ibrun -np $nprocs' RUN_CMD_FCST='ibrun -np $nprocs' RUN_CMD_POST='ibrun -np $nprocs' + +# Test Data Locations +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 0c8b7cd248..9bf525a353 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -37,6 +37,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -55,12 +57,13 @@ QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} QUEUE_FCST=${QUEUE_FCST:-"dev"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} -FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} -FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FIXLAM_NCO_BASE"} +staged_data_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="mpirun" @@ -71,12 +74,12 @@ RUN_CMD_POST="mpirun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} # Test Data Locations -TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" -TEST_COMIN="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/model_data" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 0b65597bee..d68c082370 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -578,7 +578,7 @@ def main(cla): write_summary_file(cla, data_store, file_templates) break - logging.warning(f'Requested files are unavialable from {data_store}') + logging.warning(f'Requested files are unavailable from {data_store}') if unavailable: logging.error('Could not find any of the requested files.') diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 725759dc99..12b11e88da 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -6,23 +6,6 @@ #----------------------------------------------------------------------- # function set_extrn_mdl_params() { - # - #----------------------------------------------------------------------- - # - # Use known locations or COMIN as default, depending on RUN_ENVIR - # - #----------------------------------------------------------------------- - # - if [ "${RUN_ENVIR}" = "nco" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMIN}" - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMIN}" - else - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$(set_known_sys_dir \ - ${EXTRN_MDL_NAME_ICS})}" - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$(set_known_sys_dir \ - ${EXTRN_MDL_NAME_LBCS})}" - fi - # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index d9bdffdf11..67b6776926 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -507,7 +507,7 @@ if [ -z "${NCORES_PER_NODE:-}" ]; then fi if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then - print_err_msg_exit "\ + print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} @@ -515,7 +515,7 @@ One or more fix file directories have not been specified for this machine: FIXlut = \"${FIXlut:-\"\"} TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR:-\"\"} You can specify the missing location(s) in ${machine_file}" fi @@ -1352,14 +1352,15 @@ USE_USER_STAGED_EXTRN_FILES=$(boolify $USE_USER_STAGED_EXTRN_FILES) # if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_ICS}" ]; then + # Check for the base directory up to the first templated field. + if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_ICS%%\$*})" ]; then print_err_msg_exit "\ The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: EXTRN_MDL_SOURCE_BASEDIR_ICS = \"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\"" fi - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_LBCS}" ]; then + if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_LBCS%%\$*})" ]; then print_err_msg_exit "\ The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: @@ -1474,7 +1475,7 @@ LOAD_MODULES_RUN_TASK_FP="$USHDIR/load_modules_run_task.sh" # if [ "${RUN_ENVIR}" = "nco" ]; then - nco_fix_dir="${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME}" + nco_fix_dir="${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME}" if [ ! -d "${nco_fix_dir}" ]; then print_err_msg_exit "\ The directory (nco_fix_dir) that should contain the pregenerated grid, @@ -1490,11 +1491,11 @@ orography, and surface climatology files does not exist: When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated grid files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must @@ -1528,11 +1529,11 @@ Reset values are: msg=" When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated orography files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must @@ -1567,11 +1568,11 @@ Reset values are: When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated surface climatology files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index d82eeec677..ef5d49c8e7 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -64,14 +64,12 @@ FV3GFS: - gfs.t{hh}z.sfcanl.nemsio fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio - - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio netcdf: anl: - gfs.t{hh}z.atmanl.nc - gfs.t{hh}z.sfcanl.nc fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nc - - gfs.t{hh}z.sfcf{fcst_hr:03d}.nc hpss: protocol: htar archive_path: