Merge remote-tracking branch 'upstream/develop' into feature/move_jcb

* upstream/develop:
  Add Hercules-EMC to the Jenkins configurable parameter list (NOAA-EMC#2685)
  Update gdas.cd and gsi_utils hashes (NOAA-EMC#2641)
  Add ability to use GEFS replay ICs (NOAA-EMC#2559)
  Replace `sleep` with `wait_for_file` (NOAA-EMC#2586)

ci/Jenkinsfile

@@ -76,7 +76,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC' 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
                         sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)

env/HERA.env

@@ -316,7 +316,6 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
 
-
 elif [[ "${step}" = "fit2obs" ]]; then
 
     nth_max=$((npe_node_max / npe_node_fit2obs))

parm/config/gefs/config.base

@@ -294,12 +294,17 @@ export MEMDIR="mem${ENSMEM}"
 
 # initialize ocean ensemble members with perturbations
 #   if true, only occurs for members greater than zero
-export USE_OCN_PERTURB_FILES=@USE_OCN_PERTURB_FILES@
+export REPLAY_ICS=@REPLAY_ICS@
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+    export OFFSET_START_HOUR=$(( assim_freq / 2 ))
+else
+    export OFFSET_START_HOUR=0
+fi
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
-  export IAU_FHROT=0
+  export IAU_FHROT=${OFFSET_START_HOUR}
 else
   if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3

parm/config/gefs/config.efcs

@@ -67,7 +67,7 @@ export EPBL="0.8,0.4,0.2,0.08,0.04"
 export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
 export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 
-if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
     export ODA_INCUPD="True"
     export ODA_TEMPINC_VAR='t_pert'
     export ODA_SALTINC_VAR='s_pert'

parm/config/gefs/config.fcst

@@ -256,6 +256,9 @@ else
   export io_layout="1,1"
 fi
 
+if (( OFFSET_START_HOUR != 0 )); then
+    export reforecast="YES"
+fi
 # Remember config.efcs will over-ride these values for ensemble forecasts
 # if these variables are re-defined there.
 # Otherwise, the ensemble forecast will inherit from config.fcst

parm/config/gefs/config.stage_ic

@@ -13,12 +13,21 @@ case "${CASE}" in
     export CPL_ICEIC=""
     export CPL_OCNIC=""
     export CPL_WAVIC=""
+    export CPL_MEDIC=""
+    ;;
+  "C96")
+    export CPL_ATMIC=""
+    export CPL_ICEIC=""
+    export CPL_OCNIC=""
+    export CPL_WAVIC=""
+    export CPL_MEDIC=""
     ;;
   "C48")
     export CPL_ATMIC="gefs_test"
     export CPL_ICEIC="gefs_test"
     export CPL_OCNIC="gefs_test"
     export CPL_WAVIC="gefs_test"
+    export CPL_MEDIC="gefs_test"
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

parm/config/gefs/config.ufs

@@ -324,7 +324,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       MOM6_RESTART_SETTING='r'
       MOM6_RIVER_RUNOFF='False'
       eps_imesh="2.5e-1"
-      TOPOEDITS="ufs.topo_edits_011818.nc"
+      TOPOEDITS="topo_edits_011818.nc"
       if [[ "${DO_JEDIOCNVAR:-NO}" = "YES" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
         MOM6_DIAG_MISVAL="0.0"

parm/config/gefs/yaml/defaults.yaml

@@ -9,5 +9,5 @@ base:
   DO_AWIPS: "NO"
   KEEPDATA: "NO" 
   FHMAX_GFS: 120
-  USE_OCN_PERTURB_FILES: "false"
+  REPLAY_ICS: "NO"
 

parm/config/gfs/config.base

@@ -470,6 +470,10 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
    export DO_VMINMON="NO"     # GSI minimization monitoring
 fi
 
+# If starting ICs that are not at cycle hour
+export REPLAY_ICS="NO"
+export OFFSET_START_HOUR=0
+
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 

scripts/exgfs_atmos_awips_20km_1p0deg.sh

@@ -43,20 +43,14 @@ source "${USHgfs}/product_functions.sh"
 ###############################################
 # Wait for the availability of the pgrb file
 ###############################################
-icnt=1
-while (( icnt < 1000 )); do
-   if [[ -s "${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx" ]]; then
-      break
-   fi
-
-   sleep 10
-   icnt=$((icnt + 1))
-   if (( icnt >= 180 )); then
-      msg="FATAL ERROR: No GFS pgrb2 file after 30 min of waiting"
-      err_exit "${msg}"
-      exit 5
-   fi
-done
+sleep_interval=10
+max_tries=180
+idxfile="${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx"
+if ! wait_for_file "${idxfile}" "${sleep_interval}" "${max_tries}"; then
+  msg="FATAL ERROR: No GFS pgrb2 file after waiting"
+  err_exit "${msg}"
+  exit 5
+fi
 
 ########################################
 

scripts/exgfs_atmos_postsnd.sh

@@ -50,6 +50,8 @@ declare -x LEVS
 
 ### Loop for the hour and wait for the sigma and surface flux file:
 export FSTART=$STARTHOUR
+sleep_interval=10
+max_tries=360
 #
 while [ $FSTART -lt $ENDHOUR ]
 do
@@ -69,28 +71,18 @@ export FINT=$NINT1
        export MAKEBUFR=YES
    fi
 
-   ic=0
-   while [ $ic -lt 1000 ]; do
-      if [[ ! -f "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}" ]]; then
-          sleep 10
-          ic=$(expr $ic + 1)
-      else
-          break
-      fi
-
-      if [ $ic -ge 360 ]
-      then
-         err_exit "COULD NOT LOCATE logf$FEND file AFTER 1 HOUR"
-      fi
-   done
+   filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}"
+   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then
+     err_exit "FATAL ERROR: logf${FEND} not found after waiting $((sleep_interval * ( max_tries - 1) )) secs"
+   fi
 
 ## 1-hourly output before $NEND1, 3-hourly output after
-   if [ $FEND -gt $NEND1 ]; then
+   if [[ $((10#$FEND)) -gt $((10#$NEND1)) ]]; then
      export FINT=$NINT3
    fi
    ${USHgfs}/gfs_bufr.sh
 
-   export FSTART=$FEND
+   export FSTART="${FEND}"
 done
 
 ##############################################################

scripts/exgfs_wave_nawips.sh

@@ -44,6 +44,7 @@ pdsext=no
 g2tbls=g2varswmo2.tbl
 NAGRIB=nagrib2
 
+sleep_interval=20
 maxtries=15
 fhcnt=${fstart}
 while [ ${fhcnt} -le ${FHMAX_WAV} ]; do
@@ -72,28 +73,11 @@ while [ ${fhcnt} -le ${FHMAX_WAV} ]; do
     esac
     GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdIDin}.f${fhr}.grib2"
     GRIBIN_chk=${GRIBIN}.idx
-
-    icnt=1
-    while [ ${icnt} -lt 1000 ]; do
-      if [ -r ${GRIBIN_chk} ] ; then
-        break
-      else
-        let "icnt=icnt+1"
-        sleep 20
-      fi
-      if [ ${icnt} -ge ${maxtries} ]; then
-        msg="ABORTING after 5 minutes of waiting for ${GRIBIN}."
-        echo ' '
-        echo '**************************** '
-        echo '*** ERROR : NO GRIB FILE *** '
-        echo '**************************** '
-        echo ' '
-        echo ${msg}
-        set_trace
-        echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> ${wavelog}
-        err=1;export err;${errchk} || exit ${err}
-      fi
-    done
+    if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${maxtries}"; then
+      echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs"
+      echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> "${wavelog}"
+      err=1;export err;"${errchk}" || exit "${err}"
+    fi
 
     #if [ "$grdIDin" = "global.0p25" && "$grid" = "glo_30m" ]; then
     if [ "${grdIDin}" = "global.0p25" ]; then

scripts/exgfs_wave_post_gridded_sbs.sh

@@ -231,6 +231,7 @@ source "${USHgfs}/preamble.sh"
     fhr=$FHMIN_WAV
   fi
   fhrg=$fhr
+  sleep_interval=10
   iwaitmax=120 # Maximum loop cycles for waiting until wave component output file is ready (fails after max)
   while [ $fhr -le $FHMAX_WAV ]; do
 
@@ -253,26 +254,21 @@ source "${USHgfs}/preamble.sh"
     export GRDIDATA=${DATA}/output_$YMDHMS
 
 # Gridded data (main part, need to be run side-by-side with forecast
-
+    
     if [ $fhr = $fhrg ]
     then
-      iwait=0
-      for wavGRD in ${waveGRD} ; do
-        gfile=${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS}
-        while [ ! -s ${gfile} ]; do sleep 10; let iwait=iwait+1; done
-        if [ $iwait -eq $iwaitmax ]; then
-          echo '*************************************************** '
-          echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.$grdID "
-          echo '*************************************************** '
-          echo ' '
-          set_trace
+
+      for wavGRD in ${waveGRD}; do
+        gfile="${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS}"
+        if ! wait_for_file "${gfile}" "${sleep_interval}" "${iwaitmax}"; then
+          echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.${grdID}"
           echo "${WAV_MOD_TAG} post ${grdID} ${PDY} ${cycle} : field output missing."
-          err=3; export err;${errchk}
-          exit $err
+          err=3; export err; "${errchk}"
+          exit "${err}"
         fi
-        ${NLN} ${gfile} ./out_grd.${wavGRD}
+        ${NLN} "${gfile}" "./out_grd.${wavGRD}"
       done
-
+      
       if [ "$DOGRI_WAV" = 'YES' ]
       then
         nigrd=1

scripts/exgfs_wave_post_pnt.sh

@@ -247,8 +247,9 @@ source "${USHgfs}/preamble.sh"
         -e "s/FORMAT/F/g" \
                                ww3_outp_spec.inp.tmpl > ww3_outp.inp
 
-    ${NLN} mod_def.$waveuoutpGRD mod_def.ww3
-    HMS="${cyc}0000"
+    ${NLN} mod_def.${waveuoutpGRD} mod_def.ww3
+    HH=$(date --utc -d "${PDY:0:8} ${cyc} + ${FHMIN_WAV} hours" +%H)
+    HMS="${HH}0000"
     if [[ -f "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" ]]; then
       ${NLN} "${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" \
         "./out_pnt.${waveuoutpGRD}"

scripts/exgfs_wave_prdgen_gridded.sh

@@ -96,30 +96,14 @@ grids=${grids:-ak_10m at_10m ep_10m wc_10m glo_30m}
      #
 
      GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdID}.f${fhr}.grib2"
-     GRIBIN_chk=$GRIBIN.idx
-
-     icnt=1
-     while [ $icnt -lt 1000 ]; do
-       if [ -r $GRIBIN_chk ] ; then
-         break
-       else
-         echo "Waiting for input file: $GRIBIN"
-         let "icnt=icnt+1"
-         sleep 5
-       fi
-       if [ $icnt -ge $maxtries ]; then
-         msg="ABNORMAL EXIT: NO GRIB FILE FOR GRID $GRIBIN"
-         echo ' '
-         echo '**************************** '
-         echo '*** ERROR : NO GRIB FILE *** '
-         echo '**************************** '
-         echo ' '
-         echo $msg
-         set_trace
-         echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog
-         err=1;export err;${errchk} || exit ${err}
-       fi
-     done
+     GRIBIN_chk="${GRIBIN}.idx"
+     sleep_interval=5
+     max_tries=1000
+     if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${max_tries}"; then
+       echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs"
+       echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog
+       err=1;export err;${errchk} || exit ${err}
+     fi
 
      GRIBOUT=$RUNwave.$cycle.$grdID.f${fhr}.clipped.grib2