diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 317cb7d820..263de10c6d 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -76,7 +76,7 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC' 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) HOME = "${WORKSPACE}" sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) diff --git a/env/HERA.env b/env/HERA.env index ccaaea32e7..2157e90031 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -316,7 +316,6 @@ elif [[ "${step}" = "gempak" ]]; then export NTHREADS_GEMPAK=${nth_gempak:-1} [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max} - elif [[ "${step}" = "fit2obs" ]]; then nth_max=$((npe_node_max / npe_node_fit2obs)) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 840f33ad60..0bb29e31ae 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -294,12 +294,17 @@ export MEMDIR="mem${ENSMEM}" # initialize ocean ensemble members with perturbations # if true, only occurs for members greater than zero -export USE_OCN_PERTURB_FILES=@USE_OCN_PERTURB_FILES@ +export REPLAY_ICS=@REPLAY_ICS@ +if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + export OFFSET_START_HOUR=$(( assim_freq / 2 )) +else + export OFFSET_START_HOUR=0 +fi export DOIAU="NO" # While we are not doing IAU, we may want to warm start w/ IAU in the future # Check if cycle is cold starting if [[ "${EXP_WARM_START}" = ".false." ]]; then - export IAU_FHROT=0 + export IAU_FHROT=${OFFSET_START_HOUR} else if [[ "${DOIAU}" = "YES" ]]; then export IAU_FHROT=3 diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index 915726b974..807ed66d48 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -67,7 +67,7 @@ export EPBL="0.8,0.4,0.2,0.08,0.04" export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7" export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3" -if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then +if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then export ODA_INCUPD="True" export ODA_TEMPINC_VAR='t_pert' export ODA_SALTINC_VAR='s_pert' diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 0009f4a868..103d6f091d 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -256,6 +256,9 @@ else export io_layout="1,1" fi +if (( OFFSET_START_HOUR != 0 )); then + export reforecast="YES" +fi # Remember config.efcs will over-ride these values for ensemble forecasts # if these variables are re-defined there. # Otherwise, the ensemble forecast will inherit from config.fcst diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index b332ee1826..f0b5dfa609 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -13,12 +13,21 @@ case "${CASE}" in export CPL_ICEIC="" export CPL_OCNIC="" export CPL_WAVIC="" + export CPL_MEDIC="" + ;; + "C96") + export CPL_ATMIC="" + export CPL_ICEIC="" + export CPL_OCNIC="" + export CPL_WAVIC="" + export CPL_MEDIC="" ;; "C48") export CPL_ATMIC="gefs_test" export CPL_ICEIC="gefs_test" export CPL_OCNIC="gefs_test" export CPL_WAVIC="gefs_test" + export CPL_MEDIC="gefs_test" ;; *) echo "FATAL ERROR Unrecognized resolution: ${CASE}" diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 9c39bf06de..8beb0652f7 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -324,7 +324,7 @@ if [[ "${skip_mom6}" == "false" ]]; then MOM6_RESTART_SETTING='r' MOM6_RIVER_RUNOFF='False' eps_imesh="2.5e-1" - TOPOEDITS="ufs.topo_edits_011818.nc" + TOPOEDITS="topo_edits_011818.nc" if [[ "${DO_JEDIOCNVAR:-NO}" = "YES" ]]; then MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" MOM6_DIAG_MISVAL="0.0" diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index 5c763ad29e..2341c35d05 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -9,5 +9,5 @@ base: DO_AWIPS: "NO" KEEPDATA: "NO" FHMAX_GFS: 120 - USE_OCN_PERTURB_FILES: "false" + REPLAY_ICS: "NO" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 7eedec8af7..6cc1b6d744 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -470,6 +470,10 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VMINMON="NO" # GSI minimization monitoring fi +# If starting ICs that are not at cycle hour +export REPLAY_ICS="NO" +export OFFSET_START_HOUR=0 + # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} diff --git a/scripts/exgfs_atmos_awips_20km_1p0deg.sh b/scripts/exgfs_atmos_awips_20km_1p0deg.sh index 490875b2c4..4959bbd8e8 100755 --- a/scripts/exgfs_atmos_awips_20km_1p0deg.sh +++ b/scripts/exgfs_atmos_awips_20km_1p0deg.sh @@ -43,20 +43,14 @@ source "${USHgfs}/product_functions.sh" ############################################### # Wait for the availability of the pgrb file ############################################### -icnt=1 -while (( icnt < 1000 )); do - if [[ -s "${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx" ]]; then - break - fi - - sleep 10 - icnt=$((icnt + 1)) - if (( icnt >= 180 )); then - msg="FATAL ERROR: No GFS pgrb2 file after 30 min of waiting" - err_exit "${msg}" - exit 5 - fi -done +sleep_interval=10 +max_tries=180 +idxfile="${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx" +if ! wait_for_file "${idxfile}" "${sleep_interval}" "${max_tries}"; then + msg="FATAL ERROR: No GFS pgrb2 file after waiting" + err_exit "${msg}" + exit 5 +fi ######################################## diff --git a/scripts/exgfs_atmos_postsnd.sh b/scripts/exgfs_atmos_postsnd.sh index 23c41157fe..caf5443a50 100755 --- a/scripts/exgfs_atmos_postsnd.sh +++ b/scripts/exgfs_atmos_postsnd.sh @@ -50,6 +50,8 @@ declare -x LEVS ### Loop for the hour and wait for the sigma and surface flux file: export FSTART=$STARTHOUR +sleep_interval=10 +max_tries=360 # while [ $FSTART -lt $ENDHOUR ] do @@ -69,28 +71,18 @@ export FINT=$NINT1 export MAKEBUFR=YES fi - ic=0 - while [ $ic -lt 1000 ]; do - if [[ ! -f "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}" ]]; then - sleep 10 - ic=$(expr $ic + 1) - else - break - fi - - if [ $ic -ge 360 ] - then - err_exit "COULD NOT LOCATE logf$FEND file AFTER 1 HOUR" - fi - done + filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}" + if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then + err_exit "FATAL ERROR: logf${FEND} not found after waiting $((sleep_interval * ( max_tries - 1) )) secs" + fi ## 1-hourly output before $NEND1, 3-hourly output after - if [ $FEND -gt $NEND1 ]; then + if [[ $((10#$FEND)) -gt $((10#$NEND1)) ]]; then export FINT=$NINT3 fi ${USHgfs}/gfs_bufr.sh - export FSTART=$FEND + export FSTART="${FEND}" done ############################################################## diff --git a/scripts/exgfs_wave_nawips.sh b/scripts/exgfs_wave_nawips.sh index 69c4e54ebb..949425cbc1 100755 --- a/scripts/exgfs_wave_nawips.sh +++ b/scripts/exgfs_wave_nawips.sh @@ -44,6 +44,7 @@ pdsext=no g2tbls=g2varswmo2.tbl NAGRIB=nagrib2 +sleep_interval=20 maxtries=15 fhcnt=${fstart} while [ ${fhcnt} -le ${FHMAX_WAV} ]; do @@ -72,28 +73,11 @@ while [ ${fhcnt} -le ${FHMAX_WAV} ]; do esac GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdIDin}.f${fhr}.grib2" GRIBIN_chk=${GRIBIN}.idx - - icnt=1 - while [ ${icnt} -lt 1000 ]; do - if [ -r ${GRIBIN_chk} ] ; then - break - else - let "icnt=icnt+1" - sleep 20 - fi - if [ ${icnt} -ge ${maxtries} ]; then - msg="ABORTING after 5 minutes of waiting for ${GRIBIN}." - echo ' ' - echo '**************************** ' - echo '*** ERROR : NO GRIB FILE *** ' - echo '**************************** ' - echo ' ' - echo ${msg} - set_trace - echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> ${wavelog} - err=1;export err;${errchk} || exit ${err} - fi - done + if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${maxtries}"; then + echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs" + echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> "${wavelog}" + err=1;export err;"${errchk}" || exit "${err}" + fi #if [ "$grdIDin" = "global.0p25" && "$grid" = "glo_30m" ]; then if [ "${grdIDin}" = "global.0p25" ]; then diff --git a/scripts/exgfs_wave_post_gridded_sbs.sh b/scripts/exgfs_wave_post_gridded_sbs.sh index cee6d40b49..02aa8c456d 100755 --- a/scripts/exgfs_wave_post_gridded_sbs.sh +++ b/scripts/exgfs_wave_post_gridded_sbs.sh @@ -231,6 +231,7 @@ source "${USHgfs}/preamble.sh" fhr=$FHMIN_WAV fi fhrg=$fhr + sleep_interval=10 iwaitmax=120 # Maximum loop cycles for waiting until wave component output file is ready (fails after max) while [ $fhr -le $FHMAX_WAV ]; do @@ -253,26 +254,21 @@ source "${USHgfs}/preamble.sh" export GRDIDATA=${DATA}/output_$YMDHMS # Gridded data (main part, need to be run side-by-side with forecast - + if [ $fhr = $fhrg ] then - iwait=0 - for wavGRD in ${waveGRD} ; do - gfile=${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS} - while [ ! -s ${gfile} ]; do sleep 10; let iwait=iwait+1; done - if [ $iwait -eq $iwaitmax ]; then - echo '*************************************************** ' - echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.$grdID " - echo '*************************************************** ' - echo ' ' - set_trace + + for wavGRD in ${waveGRD}; do + gfile="${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS}" + if ! wait_for_file "${gfile}" "${sleep_interval}" "${iwaitmax}"; then + echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.${grdID}" echo "${WAV_MOD_TAG} post ${grdID} ${PDY} ${cycle} : field output missing." - err=3; export err;${errchk} - exit $err + err=3; export err; "${errchk}" + exit "${err}" fi - ${NLN} ${gfile} ./out_grd.${wavGRD} + ${NLN} "${gfile}" "./out_grd.${wavGRD}" done - + if [ "$DOGRI_WAV" = 'YES' ] then nigrd=1 diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 6e456e2aec..56cb83be21 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -247,8 +247,9 @@ source "${USHgfs}/preamble.sh" -e "s/FORMAT/F/g" \ ww3_outp_spec.inp.tmpl > ww3_outp.inp - ${NLN} mod_def.$waveuoutpGRD mod_def.ww3 - HMS="${cyc}0000" + ${NLN} mod_def.${waveuoutpGRD} mod_def.ww3 + HH=$(date --utc -d "${PDY:0:8} ${cyc} + ${FHMIN_WAV} hours" +%H) + HMS="${HH}0000" if [[ -f "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" ]]; then ${NLN} "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" \ "./out_pnt.${waveuoutpGRD}" diff --git a/scripts/exgfs_wave_prdgen_gridded.sh b/scripts/exgfs_wave_prdgen_gridded.sh index c51ce60acc..c896423ac1 100755 --- a/scripts/exgfs_wave_prdgen_gridded.sh +++ b/scripts/exgfs_wave_prdgen_gridded.sh @@ -96,30 +96,14 @@ grids=${grids:-ak_10m at_10m ep_10m wc_10m glo_30m} # GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdID}.f${fhr}.grib2" - GRIBIN_chk=$GRIBIN.idx - - icnt=1 - while [ $icnt -lt 1000 ]; do - if [ -r $GRIBIN_chk ] ; then - break - else - echo "Waiting for input file: $GRIBIN" - let "icnt=icnt+1" - sleep 5 - fi - if [ $icnt -ge $maxtries ]; then - msg="ABNORMAL EXIT: NO GRIB FILE FOR GRID $GRIBIN" - echo ' ' - echo '**************************** ' - echo '*** ERROR : NO GRIB FILE *** ' - echo '**************************** ' - echo ' ' - echo $msg - set_trace - echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog - err=1;export err;${errchk} || exit ${err} - fi - done + GRIBIN_chk="${GRIBIN}.idx" + sleep_interval=5 + max_tries=1000 + if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${max_tries}"; then + echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs" + echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog + err=1;export err;${errchk} || exit ${err} + fi GRIBOUT=$RUNwave.$cycle.$grdID.f${fhr}.clipped.grib2 diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index d941fa10b4..8800292752 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -8,6 +8,9 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) gPDY="${GDATE:0:8}" gcyc="${GDATE:8:2}" +RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) +DTG_PREFIX="${RDATE:0:8}.${RDATE:8:2}0000" + MEMDIR_ARRAY=() if [[ "${RUN:-}" = "gefs" ]]; then # Populate the member_dirs array based on the value of NMEM_ENS @@ -33,9 +36,10 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Stage the FV3 restarts to ROTDIR (warm start) RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}" - for ftype in coupler.res fv_core.res.nc; do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}" - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}" + prev_atmos_copy_list=(fv_core.res.nc coupler.res) + for ftype in "${prev_atmos_copy_list[@]}"; do + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}" + tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -43,11 +47,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do done for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do for ((tt = 1; tt <= ntiles; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}.tile${tt}.nc" if (( tt > 6 )) ; then - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc" + tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.nest0$((tt-5)).tile${tt}.nc" else - tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc" + tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.tile${tt}.nc" fi ${NCP} "${src}" "${tgt}" rc=$? @@ -79,13 +83,26 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do fi done fi + + # Atmosphere Perturbation Files (usually used with replay ICS) + # Extra zero on MEMDIR ensure we have a number even if the string is empty + if (( 0${MEMDIR:3} > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + [[ ! -d "${COM_ATMOS_ANALYSIS}" ]] && mkdir -p "${COM_ATMOS_ANALYSIS}" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.fv3_perturbation.nc" + tgt="${COM_ATMOS_ANALYSIS}/${RUN}.t00z.atminc.nc" + ${NCP} "${src}" "${tgt}" + rc=${?} + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + fi # Stage ocean initial conditions to ROTDIR (warm start) if [[ "${DO_OCN:-}" = "YES" ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res.nc" + src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res.nc" + tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res.nc" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -96,8 +113,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do ;; "025" ) for nn in $(seq 1 3); do - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res_${nn}.nc" + src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res_${nn}.nc" + tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res_${nn}.nc" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -113,9 +130,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Ocean Perturbation Files # Extra zero on MEMDIR ensure we have a number even if the string is empty - if (( 0${MEMDIR:3} > 0 )) && [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.mom6_increment.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.mom6_increment.nc" + if (( 0${MEMDIR:3} > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + YMD=${PDY} HH=${cyc} declare_from_tmpl COM_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL + [[ ! -d "${COM_OCEAN_ANALYSIS}" ]] && mkdir -p "${COM_OCEAN_ANALYSIS}" + src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.mom6_perturbation.nc" + tgt="${COM_OCEAN_ANALYSIS}/mom6_increment.nc" ${NCP} "${src}" "${tgt}" rc=${?} ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -128,8 +147,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model) RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" - tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc" + src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${DTG_PREFIX}.ufs.cpld.cpl.r.nc" + tgt="${COM_MED_RESTART_PREV}/${DTG_PREFIX}.ufs.cpld.cpl.r.nc" if [[ -f "${src}" ]]; then ${NCP} "${src}" "${tgt}" rc=$? @@ -146,8 +165,8 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do if [[ "${DO_ICE:-}" = "YES" ]]; then RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc" - tgt="${COM_ICE_RESTART_PREV}/${PDY}.${cyc}0000.cice_model.res.nc" + src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${DTG_PREFIX}.cice_model.res.nc" + tgt="${COM_ICE_RESTART_PREV}/${DTG_PREFIX}.cice_model.res.nc" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" @@ -156,11 +175,11 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc) if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_WAVE_RESTART - [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}" + YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL + [[ ! -d "${COM_WAVE_RESTART_PREV}" ]] && mkdir -p "${COM_WAVE_RESTART_PREV}" for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust - src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}" - tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}" + src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${DTG_PREFIX}.restart.${grdID}" + tgt="${COM_WAVE_RESTART_PREV}/${DTG_PREFIX}.restart.${grdID}" ${NCP} "${src}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index d940406161..4332814529 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit d9404061611553459394173c3ff33116db306326 +Subproject commit 4332814529465ab8eb58e43a38227b952ebfca49 diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index aff4f5a394..a4e10dffb9 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -8,40 +8,24 @@ FV3_postdet() { echo "warm_start = ${warm_start}" echo "RERUN = ${RERUN}" + #============================================================================ + # First copy initial conditions # cold start case if [[ "${warm_start}" == ".false." ]]; then - # Create an array of chgres-ed FV3 files - local fv3_input_files tile_files fv3_input_file - fv3_input_files=(gfs_ctrl.nc) - tile_files=(gfs_data sfc_data) - local nn tt - for (( nn = 1; nn <= ntiles; nn++ )); do - for tt in "${tile_files[@]}"; do - fv3_input_files+=("${tt}.tile${nn}.nc") - done - done - + # Get list of FV3 cold start files + local file_list + file_list=$(FV3_coldstarts) echo "Copying FV3 cold start files for 'RUN=${RUN}' at '${current_cycle}' from '${COMIN_ATMOS_INPUT}'" - for fv3_input_file in "${fv3_input_files[@]}"; do - ${NCP} "${COMIN_ATMOS_INPUT}/${fv3_input_file}" "${DATA}/INPUT/${fv3_input_file}" \ + local fv3_file + for fv3_file in ${file_list}; do + ${NCP} "${COMIN_ATMOS_INPUT}/${fv3_file}" "${DATA}/INPUT/${fv3_file}" \ || ( echo "FATAL ERROR: Unable to copy FV3 IC, ABORT!"; exit 1 ) done # warm start case elif [[ "${warm_start}" == ".true." ]]; then - # Create an array of FV3 restart files - local fv3_restart_files tile_files fv3_restart_file restart_file - fv3_restart_files=(coupler.res fv_core.res.nc) - tile_files=(fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data ca_data) - local nn tt - for (( nn = 1; nn <= ntiles; nn++ )); do - for tt in "${tile_files[@]}"; do - fv3_restart_files+=("${tt}.tile${nn}.nc") - done - done - # Determine restart date and directory containing restarts local restart_date restart_dir if [[ "${RERUN}" == "YES" ]]; then @@ -52,26 +36,18 @@ FV3_postdet() { restart_dir="${COMIN_ATMOS_RESTART_PREV}" fi + # Get list of FV3 restart files + local file_list + file_list=$(FV3_restarts) echo "Copying FV3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" - for fv3_restart_file in "${fv3_restart_files[@]}"; do - restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${fv3_restart_file}" - ${NCP} "${restart_dir}/${restart_file}" "${DATA}/INPUT/${fv3_restart_file}" \ + local fv3_file restart_file + for fv3_file in ${file_list}; do + restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${fv3_file}" + ${NCP} "${restart_dir}/${restart_file}" "${DATA}/INPUT/${fv3_file}" \ || ( echo "FATAL ERROR: Unable to copy FV3 IC, ABORT!"; exit 1 ) done - if [[ "${RERUN}" == "YES" ]]; then - - local restart_fhr - restart_fhr=$(nhour "${RERUN_DATE}" "${current_cycle}") - IAU_FHROT=$((IAU_OFFSET + restart_fhr)) - if [[ "${DOIAU}" == "YES" ]]; then - IAUFHRS=-1 - IAU_DELTHRS=0 - IAU_INC_FILES="''" - fi - - else # "${RERUN}" == "NO" - + if [[ "${RERUN}" != "YES" ]]; then # Replace sfc_data with sfcanl_data restart files from current cycle (if found) local nn for (( nn = 1; nn <= ntiles; nn++ )); do @@ -84,9 +60,55 @@ FV3_postdet() { break fi done + fi # if [[ "${RERUN}" != "YES" ]]; then - # Need a coupler.res that is consistent with the model start time + fi # if [[ "${warm_start}" == ".true." ]]; then + + #============================================================================ + # Determine increment files when doing cold start + if [[ "${warm_start}" == ".false." ]]; then + + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + IAU_FHROT=${half_window} # Replay ICs start at the end of the assimilation window + if (( MEMBER == 0 )); then + inc_files=() + else + inc_files=("atminc.nc") + read_increment=".true." + res_latlon_dynamics="atminc.nc" + fi + local increment_file + for inc_file in "${inc_files[@]}"; do + increment_file="${COMIN_ATMOS_INPUT}/${RUN}.t${cyc}z.${inc_file}" + if [[ -f "${increment_file}" ]]; then + ${NCP} "${increment_file}" "${DATA}/INPUT/${inc_file}" + else + echo "FATAL ERROR: missing increment file '${increment_file}', ABORT!" + exit 1 + fi + done + fi + + # Determine IAU and increment files when doing warm start + elif [[ "${warm_start}" == ".true." ]]; then + + #-------------------------------------------------------------------------- + if [[ "${RERUN}" == "YES" ]]; then + + local restart_fhr + restart_fhr=$(nhour "${RERUN_DATE}" "${current_cycle}") + IAU_FHROT=$((IAU_OFFSET + restart_fhr)) if [[ "${DOIAU}" == "YES" ]]; then + IAUFHRS=-1 + IAU_DELTHRS=0 + IAU_INC_FILES="''" + fi + + #-------------------------------------------------------------------------- + else # "${RERUN}" == "NO" + + # Need a coupler.res that is consistent with the model start time + if [[ "${DOIAU:-NO}" == "YES" ]]; then local model_start_time="${previous_cycle}" else local model_start_time="${current_cycle}" @@ -121,6 +143,15 @@ EOF inc_files=("atminc.nc") read_increment=".true." res_latlon_dynamics="atminc.nc" + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + IAU_FHROT=${half_window} # Replay ICs start at the end of the assimilation window + # Control member has no perturbation + if (( MEMBER == 0 )); then + inc_files=() + read_increment=".false." + res_latlon_dynamics='""' + fi + fi fi local increment_file @@ -135,17 +166,22 @@ EOF done fi # if [[ "${RERUN}" == "YES" ]]; then + #-------------------------------------------------------------------------- fi # if [[ "${warm_start}" == ".true." ]]; then + #============================================================================ + #============================================================================ # If doing IAU, change forecast hours - if [[ "${DOIAU:-}" == "YES" ]]; then + if [[ "${DOIAU:-NO}" == "YES" ]]; then FHMAX=$((FHMAX + 6)) if (( FHMAX_HF > 0 )); then FHMAX_HF=$((FHMAX_HF + 6)) fi fi + #============================================================================ + #============================================================================ # If warm starting from restart files, set the following flags if [[ "${warm_start}" == ".true." ]]; then @@ -162,8 +198,9 @@ EOF na_init=0 fi # warm_start == .true. + #============================================================================ - cd "${DATA}" || exit 1 + #============================================================================ if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then local FH2 FH3 for fhr in ${FV3_OUTPUT_FH}; do @@ -181,16 +218,8 @@ EOF fi fi done - else # TODO: Is this even valid anymore? - local nn - for (( nn = 1; nn <= ntiles; nn++ )); do - ${NLN} "nggps2d.tile${nn}.nc" "${COMOUT_ATMOS_HISTORY}/nggps2d.tile${nn}.nc" - ${NLN} "nggps3d.tile${nn}.nc" "${COMOUT_ATMOS_HISTORY}/nggps3d.tile${nn}.nc" - ${NLN} "grid_spec.tile${nn}.nc" "${COMOUT_ATMOS_HISTORY}/grid_spec.tile${nn}.nc" - ${NLN} "atmos_static.tile${nn}.nc" "${COMOUT_ATMOS_HISTORY}/atmos_static.tile${nn}.nc" - ${NLN} "atmos_4xdaily.tile${nn}.nc" "${COMOUT_ATMOS_HISTORY}/atmos_4xdaily.tile${nn}.nc" - done fi + #============================================================================ } FV3_nml() { @@ -223,44 +252,35 @@ FV3_out() { ${NCP} "${DATA}/diag_table" "${COMOUT_CONF}/ufs.diag_table" - # Create an array of fv3 restart files - local fv3_restart_files tile_files fv3_restart_file restart_file - fv3_restart_files=(coupler.res fv_core.res.nc) - tile_files=(fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data ca_data) - local nn tt - for (( nn = 1; nn <= ntiles; nn++ )); do - for tt in "${tile_files[@]}"; do - fv3_restart_files+=("${tt}.tile${nn}.nc") - done - done - + # Determine the dates for restart files to be copied to COM + local restart_date restart_dates + restart_dates=() # Copy restarts in the assimilation window for RUN=gdas|enkfgdas|enkfgfs if [[ "${RUN}" =~ "gdas" || "${RUN}" == "enkfgfs" ]]; then - local restart_date - restart_date=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${restart_interval} hours" +%Y%m%d%H) + restart_date="${model_start_date_next_cycle}" while (( restart_date <= forecast_end_cycle )); do - echo "Copying FV3 restarts for 'RUN=${RUN}' at ${restart_date}" - for fv3_restart_file in "${fv3_restart_files[@]}"; do - restart_file="${restart_date:0:8}.${restart_date:8:2}0000.${fv3_restart_file}" - ${NCP} "${DATArestart}/FV3_RESTART/${restart_file}" \ - "${COMOUT_ATMOS_RESTART}/${restart_file}" - done + restart_dates+=("${restart_date:0:8}.${restart_date:8:2}0000") restart_date=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H) done + elif [[ "${RUN}" == "gfs" || "${RUN}" == "gefs" ]]; then # Copy restarts at the end of the forecast segment for RUN=gfs|gefs + if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then + restart_dates+=("${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000") + fi fi - # Copy the final restart files at the end of the forecast segment - # The final restart written at the end of the forecast does not include the valid date - # TODO: verify the above statement since RM found that it did! - # TODO: For other components, this is only for gfs/gefs - check to see if this should also have this - if [[ "${COPY_FINAL_RESTARTS}" == "YES" ]]; then - echo "Copying FV3 restarts for 'RUN=${RUN}' at the end of the forecast segment: ${forecast_end_cycle}" - for fv3_restart_file in "${fv3_restart_files[@]}"; do - restart_file="${forecast_end_cycle:0:8}.${forecast_end_cycle:8:2}0000.${fv3_restart_file}" - ${NCP} "${DATArestart}/FV3_RESTART/${restart_file}" \ - "${COMOUT_ATMOS_RESTART}/${restart_file}" + # Get list of FV3 restart files + local file_list fv3_file + file_list=$(FV3_restarts) + + # Copy restarts for the dates collected above to COM + for restart_date in "${restart_dates[@]}"; do + echo "Copying FV3 restarts for 'RUN=${RUN}' at ${restart_date}" + for fv3_file in ${file_list}; do + ${NCP} "${DATArestart}/FV3_RESTART/${restart_date}.${fv3_file}" \ + "${COMOUT_ATMOS_RESTART}/${restart_date}.${fv3_file}" done - fi + done + echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied" } @@ -362,7 +382,7 @@ WW3_out() { CPL_out() { echo "SUB ${FUNCNAME[0]}: Copying output data for general cpl fields" - if [[ "${esmf_profile:-}" == ".true." ]]; then + if [[ "${esmf_profile:-.false.}" == ".true." ]]; then ${NCP} "${DATA}/ESMF_Profile.summary" "${COMOUT_ATMOS_HISTORY}/ESMF_Profile.summary" fi } @@ -406,7 +426,7 @@ MOM6_postdet() { # TODO if [[ $RUN} == "gefs" ]] block maybe be needed # to ensure it does not interfere with the GFS when ensemble is updated in the GFS if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_RESTART_PREV}/${restart_date:0:8}.${restart_date:0:8}0000.mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ + ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) fi fi # if [[ "${RERUN}" == "NO" ]]; then diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 9c0d90b1dc..9183e86002 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -36,6 +36,38 @@ nhour(){ echo "${hours}" } +FV3_coldstarts(){ + # Function to return an comma-separated string of cold-start input files for FV3 + # Create an array of chgres-ed FV3 files + local fv3_input_files tile_files + fv3_input_files=(gfs_ctrl.nc) + tile_files=(gfs_data sfc_data) + local nn tt + for (( nn = 1; nn <= ntiles; nn++ )); do + for tt in "${tile_files[@]}"; do + fv3_input_files+=("${tt}.tile${nn}.nc") + done + done + # Create a comma separated string from array using IFS + IFS=, echo "${fv3_input_files[*]}" +} + +FV3_restarts(){ + # Function to return an comma-separated string of warm-start input files for FV3 + # Create an array of FV3 restart files + local fv3_restart_files tile_files + fv3_restart_files=(coupler.res fv_core.res.nc) + tile_files=(fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data ca_data) + local nn tt + for (( nn = 1; nn <= ntiles; nn++ )); do + for tt in "${tile_files[@]}"; do + fv3_restart_files+=("${tt}.tile${nn}.nc") + done + done + # Create a comma separated string from array using IFS + IFS=, echo "${fv3_restart_files[*]}" +} + # shellcheck disable=SC2034 common_predet(){ echo "SUB ${FUNCNAME[0]}: Defining variables for shared through model components" @@ -54,15 +86,20 @@ common_predet(){ current_cycle_begin=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) current_cycle_end=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${half_window} hours" +%Y%m%d%H) next_cycle_begin=$(date --utc -d "${next_cycle:0:8} ${next_cycle:8:2} - ${half_window} hours" +%Y%m%d%H) - #Define model start date for current_cycle and next_cycle as the time the forecast will start - if [[ "${DOIAU:-}" == "YES" ]]; then + forecast_end_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHMAX} hours" +%Y%m%d%H) + + # Define model start date for current_cycle and next_cycle as the time the forecast will start + if [[ "${DOIAU:-NO}" == "YES" ]]; then model_start_date_current_cycle="${current_cycle_begin}" model_start_date_next_cycle="${next_cycle_begin}" else - model_start_date_current_cycle=${current_cycle} + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + model_start_date_current_cycle=${current_cycle_end} + else + model_start_date_current_cycle=${current_cycle} + fi model_start_date_next_cycle=${next_cycle} - fi - forecast_end_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHMAX} hours" +%Y%m%d%H) + fi FHMIN=${FHMIN:-0} FHMAX=${FHMAX:-9} diff --git a/ush/gfs_bufr.sh b/ush/gfs_bufr.sh index 287365ba88..8a7d9b1091 100755 --- a/ush/gfs_bufr.sh +++ b/ush/gfs_bufr.sh @@ -51,26 +51,20 @@ cat << EOF > gfsparm / EOF +sleep_interval=10 +max_tries=1000 for (( hr = 10#${FSTART}; hr <= 10#${FEND}; hr = hr + 10#${FINT} )); do hh2=$(printf %02i "${hr}") hh3=$(printf %03i "${hr}") #--------------------------------------------------------- # Make sure all files are available: - ic=0 - while (( ic < 1000 )); do - if [[ ! -f "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}" ]]; then - sleep 10 - ic=$((ic + 1)) - else - break - fi - - if (( ic >= 360 )); then - echo "FATAL: COULD NOT LOCATE logf${hh3} file AFTER 1 HOUR" - exit 2 - fi - done + filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}" + if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then + echo "FATAL ERROR: COULD NOT LOCATE logf${hh3} file" + exit 2 + fi + #------------------------------------------------------------------ ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh3}.${atmfm}" "sigf${hh2}" ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh3}.${atmfm}" "flxf${hh2}" @@ -96,11 +90,11 @@ esac ${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}" export err=$? -if [ $err -ne 0 ]; then +if [[ "${err}" -ne 0 ]]; then echo "GFS postsnd job error, Please check files " echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh2}.${atmfm}" echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh2}.${atmfm}" err_chk fi -exit ${err} +exit "${err}" diff --git a/ush/gfs_bufr_netcdf.sh b/ush/gfs_bufr_netcdf.sh index f10ba40730..f03ff3b9af 100755 --- a/ush/gfs_bufr_netcdf.sh +++ b/ush/gfs_bufr_netcdf.sh @@ -66,6 +66,9 @@ hh=$FSTART hh1=$(echo "${hh#"${hh%??}"}") hh=$hh1 fi + +sleep_interval=10 +max_tries=360 while test $hh -le $FEND do if test $hh -lt 100 @@ -75,24 +78,11 @@ do hh2=$hh fi -#--------------------------------------------------------- -# Make sure all files are available: - ic=0 - while [ $ic -lt 1000 ] - do - if [ ! -f $COMIN/${RUN}.${cycle}.logf${hh2}.txt ] - then - sleep 10 - ic=$(expr $ic + 1) - else - break - fi + filename="${COMIN}/${RUN}.${cycle}.logf${hh2}.txt" + if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}" ; then + err_exit "FATAL ERROR COULD NOT LOCATE logf${hh2} file" + fi - if [ $ic -ge 360 ] - then - err_exit "COULD NOT LOCATE logf${hh2} file AFTER 1 HOUR" - fi - done #------------------------------------------------------------------ ${NLN} $COMIN/${RUN}.${cycle}.atmf${hh2}.nc sigf${hh} ${NLN} $COMIN/${RUN}.${cycle}.${SFCF}f${hh2}.nc flxf${hh} diff --git a/ush/parsing_namelists_WW3.sh b/ush/parsing_namelists_WW3.sh index a01d694710..c57a90e50a 100755 --- a/ush/parsing_namelists_WW3.sh +++ b/ush/parsing_namelists_WW3.sh @@ -18,7 +18,7 @@ WW3_namelists(){ fi # Set time stamps for model start and output # For special case when IAU is on but this is an initial half cycle - if [ $IAU_OFFSET = 0 ]; then + if [ ${IAU_OFFSET:-0} = 0 ]; then ymdh_beg=$YMDH else ymdh_beg=$($NDATE -$WAVHINDH $YMDH) diff --git a/ush/wave_outp_spec.sh b/ush/wave_outp_spec.sh index db9997fd54..159d0eb2cf 100755 --- a/ush/wave_outp_spec.sh +++ b/ush/wave_outp_spec.sh @@ -31,6 +31,7 @@ source "${USHgfs}/preamble.sh" workdir=$4 YMDHE=$($NDATE $FHMAX_WAV_PNT $CDATE) + model_start_date=$(${NDATE} ${OFFSET_START_HOUR} "${PDY}${cyc}") cd $workdir @@ -196,7 +197,7 @@ source "${USHgfs}/preamble.sh" if [ -f $outfile ] then - if [ "${ymdh}" = "${CDATE}" ] + if [ "${ymdh}" = "${model_start_date}" ] then if [ "$specdir" = "bull" ] then @@ -237,6 +238,6 @@ source "${USHgfs}/preamble.sh" # 3.b Clean up the rest cd .. -rm -rf ${specdir}_${bloc} +rm -rf "${specdir}_${bloc}" # End of ww3_outp_spec.sh ---------------------------------------------------- # diff --git a/ush/wave_tar.sh b/ush/wave_tar.sh index bb8836df2c..e01ef61f15 100755 --- a/ush/wave_tar.sh +++ b/ush/wave_tar.sh @@ -29,7 +29,7 @@ source "${USHgfs}/preamble.sh" # 0.a Basic modes of operation - cd $DATA + cd "${DATA}" echo "Making TAR FILE" alertName=$(echo $RUN|tr [a-z] [A-Z]) @@ -47,7 +47,7 @@ source "${USHgfs}/preamble.sh" # 0.b Check if type set - if [ "$#" -lt '3' ] + if [[ "$#" -lt '3' ]] then set +x echo ' ' @@ -64,9 +64,9 @@ source "${USHgfs}/preamble.sh" fi filext=$type - if [ "$type" = "ibp" ]; then filext='spec'; fi - if [ "$type" = "ibpbull" ]; then filext='bull'; fi - if [ "$type" = "ibpcbull" ]; then filext='cbull'; fi + if [[ "$type" = "ibp" ]]; then filext='spec'; fi + if [[ "$type" = "ibpbull" ]]; then filext='bull'; fi + if [[ "$type" = "ibpcbull" ]]; then filext='cbull'; fi rm -rf TAR_${filext}_$ID @@ -88,7 +88,7 @@ source "${USHgfs}/preamble.sh" exit 2 fi - cd ${STA_DIR}/${filext} + cd "${STA_DIR}/${filext}" # --------------------------------------------------------------------------- # # 2. Generate tar file (spectral files are compressed) @@ -98,21 +98,27 @@ source "${USHgfs}/preamble.sh" echo ' Making tar file ...' set_trace - count=0 countMAX=5 tardone='no' - - while [ "$count" -lt "$countMAX" ] && [ "$tardone" = 'no' ] + sleep_interval=10 + + while [[ "${tardone}" = "no" ]] do nf=$(ls | awk '/'$ID.*.$filext'/ {a++} END {print a}') nbm2=$(( $nb - 2 )) - if [ $nf -ge $nbm2 ] - then - tar -cf $ID.$cycle.${type}_tar ./$ID.*.$filext + if [[ "${nf}" -ge "${nbm2}" ]] + then + + tar -cf "${ID}.${cycle}.${type}_tar" ./${ID}.*.${filext} exit=$? + filename="${ID}.${cycle}.${type}_tar" + if ! wait_for_file "${filename}" "${sleep_interval}" "${countMAX}" ; then + echo "FATAL ERROR: File ${filename} not found after waiting $(( sleep_interval * (countMAX + 1) )) secs" + exit 3 + fi - if [ "$exit" != '0' ] + if [[ "${exit}" != '0' ]] then set +x echo ' ' @@ -124,21 +130,15 @@ source "${USHgfs}/preamble.sh" exit 3 fi - if [ -f "$ID.$cycle.${type}_tar" ] + if [[ -f "${ID}.${cycle}.${type}_tar" ]] then tardone='yes' fi - else - set +x - echo ' All files not found for tar. Sleeping 10 seconds and trying again ..' - set_trace - sleep 10 - count=$(expr $count + 1) fi done - if [ "$tardone" = 'no' ] + if [[ "${tardone}" = 'no' ]] then set +x echo ' ' @@ -150,15 +150,15 @@ source "${USHgfs}/preamble.sh" exit 3 fi - if [ "$type" = 'spec' ] + if [[ "${type}" = 'spec' ]] then - if [ -s $ID.$cycle.${type}_tar ] + if [[ -s "${ID}.${cycle}.${type}_tar" ]] then - file_name=$ID.$cycle.${type}_tar.gz - /usr/bin/gzip -c $ID.$cycle.${type}_tar > ${file_name} + file_name="${ID}.${cycle}.${type}_tar.gz" + /usr/bin/gzip -c "${ID}.${cycle}.${type}_tar" > "${file_name}" exit=$? - if [ "$exit" != '0' ] + if [[ "${exit}" != '0' ]] then set +x echo ' ' @@ -171,7 +171,7 @@ source "${USHgfs}/preamble.sh" fi fi else - file_name=$ID.$cycle.${type}_tar + file_name="${ID}.${cycle}.${type}_tar" fi # --------------------------------------------------------------------------- # @@ -186,7 +186,7 @@ source "${USHgfs}/preamble.sh" exit=$? - if [ "$exit" != '0' ] + if [[ "${exit}" != '0' ]] then set +x echo ' ' @@ -198,7 +198,7 @@ source "${USHgfs}/preamble.sh" exit 4 fi - if [ "$SENDDBN" = 'YES' ] + if [[ "${SENDDBN}" = 'YES' ]] then set +x echo ' ' @@ -212,7 +212,7 @@ source "${USHgfs}/preamble.sh" # --------------------------------------------------------------------------- # # 4. Final clean up -cd $DATA +cd "${DATA}" if [[ ${KEEPDATA:-NO} == "NO" ]]; then set -v diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 6280e8e115..34f81bfe96 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -5,6 +5,7 @@ export spack_env=gsi-addon-dev-rocky8 export hpss_ver=hpss export ncl_ver=6.6.2 export R_ver=3.6.1 + export gempak_ver=7.17.0 export perl_ver=5.38.0 diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 99be535a55..6899a655b6 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -1,6 +1,7 @@ from applications.applications import AppConfig from rocoto.tasks import Tasks import rocoto.rocoto as rocoto +from datetime import datetime, timedelta class GEFSTasks(Tasks): @@ -9,49 +10,61 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: super().__init__(app_config, cdump) def stage_ic(self): - cpl_ic = self._configs['stage_ic'] - deps = [] - + dtg_prefix = "@Y@m@d.@H0000" + offset = str(self._configs['base']['OFFSET_START_HOUR']).zfill(2) + ":00:00" # Atm ICs if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/atmos/" + if self._base['EXP_WARM_START']: + for file in ['fv_core.res.nc'] + \ + [f'{datatype}.tile{tile}.nc' + for datatype in ['ca_data', 'fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] + for tile in range(1, self.n_tiles + 1)]: + data = [prefix, f"{dtg_prefix}.{file}"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} + deps.append(rocoto.add_dependency(dep_dict)) + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/med/" + data = [prefix, f"{dtg_prefix}.ufs.cpld.cpl.r.nc"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} deps.append(rocoto.add_dependency(dep_dict)) + else: + for file in ['gfs_ctrl.nc'] + \ + [f'{datatype}_data.tile{tile}.nc' + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: + data = f"{prefix}/{file}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) # Ocean ICs if self.app_config.do_ocean: ocn_res = f"{self._base.get('OCNRES', '025'):03d}" - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/mem000/ocean" - data = f"{prefix}/@Y@m@d.@H0000.MOM.res.nc" - dep_dict = {'type': 'data', 'data': data} + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/mem000/ocean/" + data = [prefix, f"{dtg_prefix}.MOM.res.nc"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} deps.append(rocoto.add_dependency(dep_dict)) if ocn_res in ['025']: # 0.25 degree ocean model also has these additional restarts for res in [f'res_{res_index}' for res_index in range(1, 4)]: - data = f"{prefix}/@Y@m@d.@H0000.MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} + data = [prefix, f"{dtg_prefix}.MOM.{res}.nc"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} deps.append(rocoto.add_dependency(dep_dict)) # Ice ICs if self.app_config.do_ice: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/mem000/ice" - data = f"{prefix}/@Y@m@d.@H0000.cice_model.res.nc" - dep_dict = {'type': 'data', 'data': data} + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/mem000/ice/" + data = [prefix, f"{dtg_prefix}.cice_model.res.nc"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} deps.append(rocoto.add_dependency(dep_dict)) # Wave ICs if self.app_config.do_wave: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/mem000/wave" + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/mem000/wave/" for wave_grid in self._configs['waveinit']['waveGRD'].split(): - data = f"{prefix}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} + data = [prefix, f"{dtg_prefix}.restart.{wave_grid}"] + dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 97d25dc15a..b44842b982 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -287,6 +287,8 @@ def _update_defaults(dict_in: dict) -> dict: data = AttrDict(host.info, **inputs.__dict__) data.HOMEgfs = _top yaml_path = inputs.yaml + if not os.path.exists(yaml_path): + raise IOError(f'YAML file does not exist, check path:' + yaml_path) yaml_dict = _update_defaults(AttrDict(parse_j2yaml(yaml_path, data))) # First update config.base