Merge "VLab Issue #67188 - Port to Hera" into develop

lgannoaa · Dec 5, 2019 · e4b6b7d · e4b6b7d
2 parents 3bd742d + 7b4b81c
commit e4b6b7d
Show file tree

Hide file tree

Showing 85 changed files with 2,128 additions and 21,458 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -0,0 +1,176 @@
+#!/bin/ksh -x
+
+if [ $# -ne 1 ]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    echo "argument can be any one of the following:"
+    echo "anal fcst post vrfy"
+    echo "eobs eupd ecen efcs epos"
+    echo "postsnd awips gempak"
+    exit 1
+
+fi
+
+step=$1
+
+export npe_node_max=40
+export launcher="srun --export=ALL"
+
+# Configure MPI environment
+export I_MPI_ADJUST_ALLREDUCE=5
+export MPI_BUFS_PER_PROC=2048
+export MPI_BUFS_PER_HOST=2048
+export MPI_GROUP_MAX=256
+export MPI_MEMMAP_OFF=1
+export MP_STDOUTMODE="ORDERED"
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+#export LD_BIND_NOW=1
+export job=${PBS_JOBNAME:-$step}
+export jobid=${job}.${PBS_JOBID:-$$}
+
+if [ $step = "prep" -o $step = "prepbufr" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_prep))
+
+    export POE="NO"
+    export BACK=${BACK:-"YES"}
+    export sys_tp="HERA"
+
+elif [ $step = "anal" ]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    nth_max=$(($npe_node_max / $npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher"
+
+    export NTHREADS_CYCLE=${nth_cycle:-12}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="$launcher -n $npe_cycle"
+
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+
+    export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl"
+
+elif [ $step = "eobs" ]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    nth_max=$(($npe_node_max / $npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher"
+
+elif [ $step = "eupd" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-$nth_max}
+    [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max
+    export APRUN_ENKF="$launcher"
+
+elif [ $step = "fcst" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_fcst))
+
+    export NTHREADS_FV3=${nth_fv3:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher"
+
+    export NTHREADS_REMAP=${nth_remap:-2}
+    [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max
+    export APRUN_REMAP="$launcher"
+
+elif [ $step = "efcs" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_efcs))
+
+    export NTHREADS_FV3=${nth_efcs:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher $LEVS"
+
+elif [ $step = "post" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_post))
+
+    export NTHREADS_NP=${nth_np:-1}
+    [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max
+    export APRUN_NP="$launcher"
+
+    export NTHREADS_DWN=${nth_dwn:-1}
+    [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max
+    export APRUN_DWN="$launcher"
+
+elif [ $step = "ecen" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-$nth_max}
+    [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max
+    export APRUN_ECEN="$launcher"
+
+    export NTHREADS_CHGRES=${nth_chgres:-12}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher"
+
+elif [ $step = "epos" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-$nth_max}
+    [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
+    export APRUN_EPOS="$launcher"
+
+elif [ $step = "fv3ic" ]; then
+
+    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="time"
+
+elif [ $step = "postsnd" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max
+    export APRUN_POSTSND="$launcher"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max
+    export APRUN_POSTSNDCFP="$launcher"
+
+elif [ $step = "awips" ]; then
+
+     echo "WARNING: $step is not enabled on $machine!"
+
+elif [ $step = "gempak" ]; then
+
+     echo "WARNING: $step is not enabled on $machine!"
+fi
diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env
@@ -40,7 +40,7 @@ elif [ $step = "anal" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_anal))
 
-    export NTHREADS_GSI=${nth_gsi:-$nth_max}
+    export NTHREADS_GSI=${nth_anal:-$nth_max}
     [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
     export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_anal} -N $npe_node_anal -d $NTHREADS_GSI -cc depth"
 
@@ -69,7 +69,7 @@ elif [ $step = "eobs" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_eobs))
 
-    export NTHREADS_GSI=${nth_gsi:-$nth_max}
+    export NTHREADS_GSI=${nth_anal:-$nth_max}
     [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
     export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_eobs} -N $npe_node_eobs -d $NTHREADS_GSI -cc depth"
 

diff --git a/jobs/JGFS_CYCLONE_GENESIS b/jobs/JGFS_CYCLONE_GENESIS
@@ -69,6 +69,7 @@ export CDUMP=${CDUMP:-${RUN:-"gfs"}}
 if [ $RUN_ENVIR = "nco" ]; then
     export ROTDIR=${COMROOT:?}/$NET/$envir
 fi
+export cmodel=$CDUMP
 
 ####################################
 # SENDCOM  - Copy Files From TMPDIR to $COMOUT
@@ -96,10 +97,11 @@ export COMINgfs=${COMIN}
 export COMOUT=${ROTDIR}/${RUN}.${PDY}/${cyc}
 
 export JYYYY=`echo ${PDY} | cut -c1-4`
-#export COMINgenvit=${COMINgenvit:-${COMOUT}/genesis_vital_${JYYYY}}
-#export COMOUTgenvit=${COMOUTgenvit:-${COMOUT}/genesis_vital_${JYYYY}}
-export COMINgenvit=${COMINgenvit:-${DATA}/genesis_vital_${JYYYY}}
-export COMOUTgenvit=${COMOUTgenvit:-${DATA}/genesis_vital_${JYYYY}}
+export COMINgenvit=${COMINgenvit:-${COMOUT}/genesis_vital_${JYYYY}}
+export COMOUTgenvit=${COMOUTgenvit:-${COMOUT}/genesis_vital_${JYYYY}}
+#export COMINgenvit=${COMINgenvit:-${DATA}/genesis_vital_${JYYYY}}
+#export COMOUTgenvit=${COMOUTgenvit:-${DATA}/genesis_vital_${JYYYY}}
+
 export COMINsyn=${COMINsyn:-$(compath.py arch/prod/syndat)}
 
 mkdir -m 775 -p $COMOUTgenvit

diff --git a/jobs/JGFS_CYCLONE_TRACKER b/jobs/JGFS_CYCLONE_TRACKER
@@ -126,24 +126,24 @@ export cmodel=$CDUMP
 export loopnum=1
 
 #-----------input data checking -----------------
-${USHens_tracker}/data_check.sh
+${USHens_tracker}/data_check_gfs.sh
 # exit code 6 = missing data of opportunity
 if [ $? -eq 6 ]; then exit; fi
 #------------------------------------------------
 
-mac=`echo ${SITE}`
-if [ $mac = TIDE -o $mac = GYRE ] ; then # For WCOSS
+machine=${machine:-`echo ${SITE}`}
+if [ $machine = TIDE -o $machine = GYRE ] ; then # For WCOSS
   machine=wcoss
-  ${USHens_tracker}/extrkr_g2.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} 
-elif [ $mac = LUNA -o $mac = SURGE ] ; then # For CRAY
+  ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} 
+elif [ $machine = LUNA -o $machine = SURGE -o $machine = WCOSS_C ] ; then # For CRAY
   machine=cray
-  ${APRUNTRACK} ${USHens_tracker}/extrkr_g2.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} 
-elif [ $mac = VENUS -o $mac = MARS ] ; then # For DELL
+  ${APRUNTRACK} ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA} 
+elif [ $machine = VENUS -o $machine = MARS -o $machine = WCOSS_DELL_P3 ] ; then # For DELL
   machine=dell
-  mpirun -n 1 ${USHens_tracker}/extrkr_g2.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA}
-else       # For THEIA
-  machine=theia
-  ${USHens_tracker}/extrkr_g2.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA}
+  mpirun -n 1 ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA}
+elif [ $machine = HERA ]; then # For HERA
+  machine=hera
+  ${USHens_tracker}/extrkr_gfs.sh ${loopnum} ${cmodel} ${CDATE} ${pert} ${DATA}
 fi
 export err=$?; err_chk
 

diff --git a/jobs/rocoto/fv3ic.sh b/jobs/rocoto/fv3ic.sh
@@ -37,15 +37,10 @@ status=$?
 export DATA="$RUNDIR/$CDATE/$CDUMP/fv3ic$$"
 [[ -d $DATA ]] && rm -rf $DATA
 
-# Input GFS initial condition files
-export INIDIR="$ICSDIR/$CDATE/$CDUMP"
-export ATMANL="$ICSDIR/$CDATE/$CDUMP/siganl.${CDUMP}.$CDATE"
-export SFCANL="$ICSDIR/$CDATE/$CDUMP/sfcanl.${CDUMP}.$CDATE"
-if [ -f $ICSDIR/$CDATE/$CDUMP/nstanl.${CDUMP}.$CDATE ]; then
-    export NSTANL="$ICSDIR/$CDATE/$CDUMP/nstanl.${CDUMP}.$CDATE"
-fi
+# Input GFS initial condition directory
+export INIDIR="$ICSDIR/$CDATE/$CDUMP/$CDUMP.$PDY/$cyc"
 
-# Output FV3 initial condition files
+# Output FV3 initial condition directory
 export OUTDIR="$ICSDIR/$CDATE/$CDUMP/$CASE/INPUT"
 
 export OMP_NUM_THREADS_CH=$NTHREADS_CHGRES