forked from NOAA-EMC/global-workflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/develop' into feature/dev-wcos…
…s2-resources * upstream/develop: Fix companion ocean resolution for C48 (NOAA-EMC#1066) Add trailing slash for gldas topo path (NOAA-EMC#1064) Limit number of CPU for post (NOAA-EMC#1061) Fix eupd trace (NOAA-EMC#1057) Port to S4 (NOAA-EMC#1023) Update to obsproc.v1.0.2 and prepobs.v1.0.1 (NOAA-EMC#1049) Add GDAS to the partial build list (NOAA-EMC#1050) Fix group number being treated as octal in gdas arch (NOAA-EMC#1053) Remove trace from link script (NOAA-EMC#1046)
- Loading branch information
Showing
42 changed files
with
793 additions
and
249 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,295 @@ | ||
| #!/bin/bash -x | ||
|
|
||
| if [ $# -ne 1 ]; then | ||
|
|
||
| echo "Must specify an input argument to set runtime environment variables!" | ||
| echo "argument can be any one of the following:" | ||
| echo "atmanalrun atmensanalrun" | ||
| echo "anal sfcanl fcst post vrfy metp" | ||
| echo "eobs eupd ecen efcs epos" | ||
| echo "postsnd awips gempak" | ||
| exit 1 | ||
|
|
||
| fi | ||
|
|
||
| step=$1 | ||
| PARTITION_BATCH=${PARTITION_BATCH:-"s4"} | ||
|
|
||
| if [[ ${PARTITION_BATCH} = "s4" ]]; then | ||
| export npe_node_max=32 | ||
| elif [[ ${PARTITION_BATCH} = "ivy" ]]; then | ||
| export npe_node_max=20 | ||
| fi | ||
| export launcher="srun -l --export=ALL" | ||
|
|
||
| # Configure MPI environment | ||
| export OMP_STACKSIZE=2048000 | ||
| export NTHSTACK=1024000000 | ||
|
|
||
| ulimit -s unlimited | ||
| ulimit -a | ||
|
|
||
| export job=${PBS_JOBNAME:-${step}} | ||
| export jobid=${job}.${PBS_JOBID:-$$} | ||
|
|
||
|
|
||
| if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then | ||
|
|
||
| npe_node_prep=${npe_node_prep:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_prep)) | ||
|
|
||
| export POE="NO" | ||
| export BACK="NO" | ||
| export sys_tp="S4" | ||
| export launcher_PREP="srun" | ||
|
|
||
| elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then | ||
|
|
||
| export mpmd="--multi-prog" | ||
| export CFP_MP="YES" | ||
| if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi | ||
| export wavempexec=${launcher} | ||
| export wave_mpmd=${mpmd} | ||
|
|
||
| elif [[ ${step} = "atmanalrun" ]]; then | ||
|
|
||
| export CFP_MP=${CFP_MP:-"YES"} | ||
| export USE_CFP=${USE_CFP:-"YES"} | ||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_atmanalrun=${npe_node_atmanalrun:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_atmanalrun)) | ||
|
|
||
| export NTHREADS_ATMANAL=${nth_atmanalrun:-${nth_max}} | ||
| [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max} | ||
| export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun:-0}" | ||
|
|
||
| elif [[ ${step} = "atmensanalrun" ]]; then | ||
|
|
||
| export CFP_MP=${CFP_MP:-"YES"} | ||
| export USE_CFP=${USE_CFP:-"YES"} | ||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_atmensanalrun=${npe_node_atmensanalrun:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_atmensanalrun)) | ||
|
|
||
| export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}} | ||
| [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max} | ||
| export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun:-0}" | ||
|
|
||
| elif [[ ${step} = "aeroanlrun" ]]; then | ||
|
|
||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_aeroanlrun=${npe_node_aeroanlrun:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_aeroanlrun)) | ||
|
|
||
| export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} | ||
| [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} | ||
| export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun:-0}" | ||
|
|
||
| elif [[ ${step} = "anal" || ${step} = "analcalc" ]]; then | ||
|
|
||
| export MKL_NUM_THREADS=4 | ||
| export MKL_CBWR=AUTO | ||
|
|
||
| export CFP_MP=${CFP_MP:-"YES"} | ||
| export USE_CFP=${USE_CFP:-"YES"} | ||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_anal=${npe_node_anal:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_anal)) | ||
|
|
||
| export NTHREADS_GSI=${nth_anal:-${nth_max}} | ||
| [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max} | ||
| export APRUN_GSI=${launcher} | ||
|
|
||
| export NTHREADS_CALCINC=${nth_calcinc:-1} | ||
| [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} | ||
| export APRUN_CALCINC=${launcher} | ||
|
|
||
| export NTHREADS_CYCLE=${nth_cycle:-12} | ||
| [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} | ||
| npe_cycle=${ntiles:-6} | ||
| export APRUN_CYCLE="${launcher} -n ${npe_cycle:-0}" | ||
|
|
||
|
|
||
| export NTHREADS_GAUSFCANL=1 | ||
| npe_gausfcanl=${npe_gausfcanl:-1} | ||
| export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl:-0}" | ||
|
|
||
| elif [[ ${step} = "sfcanl" ]]; then | ||
| npe_node_sfcanl=${npe_node_sfcanl:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_sfcanl)) | ||
|
|
||
| export NTHREADS_CYCLE=${nth_sfcanl:-14} | ||
| [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} | ||
| npe_sfcanl=${ntiles:-6} | ||
| export APRUN_CYCLE="${launcher} -n ${npe_sfcanl:-0}" | ||
|
|
||
| elif [[ ${step} = "gldas" ]]; then | ||
|
|
||
| npe_node_gldas=${npe_node_gldas:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_gldas)) | ||
|
|
||
| export NTHREADS_GLDAS=${nth_gldas:-${nth_max}} | ||
| [[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max} | ||
| export APRUN_GLDAS="${launcher} -n ${npe_gldas:-0}" | ||
|
|
||
| export NTHREADS_GAUSSIAN=${nth_gaussian:-1} | ||
| [[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max} | ||
| export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian:-0}" | ||
|
|
||
| # Must run data processing with exactly the number of tasks as time | ||
| # periods being processed. | ||
|
|
||
| gldas_spinup_hours=${gldas_spinup_hours:-0} | ||
| npe_gldas_data_proc=$((gldas_spinup_hours + 12)) | ||
| export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} --multi-prog" | ||
|
|
||
| elif [[ ${step} = "eobs" ]]; then | ||
|
|
||
| export MKL_NUM_THREADS=4 | ||
| export MKL_CBWR=AUTO | ||
|
|
||
| export CFP_MP=${CFP_MP:-"YES"} | ||
| export USE_CFP=${USE_CFP:-"YES"} | ||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_eobs=${npe_node_eobs:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_eobs)) | ||
|
|
||
| export NTHREADS_GSI=${nth_eobs:-${nth_max}} | ||
| [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max} | ||
| export APRUN_GSI=${launcher} | ||
|
|
||
| elif [[ ${step} = "eupd" ]]; then | ||
|
|
||
| export CFP_MP=${CFP_MP:-"YES"} | ||
| export USE_CFP=${USE_CFP:-"YES"} | ||
| export APRUNCFP="${launcher} -n \$ncmd --multi-prog" | ||
|
|
||
| npe_node_eupd=${npe_node_eupd:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_eupd)) | ||
|
|
||
| export NTHREADS_ENKF=${nth_eupd:-${nth_max}} | ||
| [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max} | ||
| export APRUN_ENKF=${launcher} | ||
|
|
||
| elif [[ ${step} = "fcst" ]]; then | ||
|
|
||
| #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs | ||
| if [[ ${CDUMP:-gdas} == "gfs" ]]; then | ||
| npe_fcst=${npe_fcst_gfs:-0} | ||
| npe_node_fcst=${npe_node_fcst_gfs:-${npe_node_max}} | ||
| nth_fv3=${nth_fv3_gfs:-1} | ||
| fi | ||
|
|
||
| npe_node_fcst=${npe_node_fcst:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_fcst)) | ||
|
|
||
| export NTHREADS_FV3=${nth_fv3:-${nth_max}} | ||
| [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max} | ||
| export cores_per_node=${npe_node_max} | ||
| export APRUN_FV3="${launcher} -n ${npe_fcst:-0}" | ||
|
|
||
| export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} | ||
| [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max} | ||
| export APRUN_REGRID_NEMSIO=${launcher} | ||
|
|
||
| export NTHREADS_REMAP=${nth_remap:-2} | ||
| [[ ${NTHREADS_REMAP} -gt ${nth_max} ]] && export NTHREADS_REMAP=${nth_max} | ||
| export APRUN_REMAP=${launcher} | ||
| export I_MPI_DAPL_UD="enable" | ||
|
|
||
| elif [[ ${step} = "efcs" ]]; then | ||
|
|
||
| npe_node_efcs=${npe_node_efcs:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_efcs)) | ||
|
|
||
| export NTHREADS_FV3=${nth_efcs:-${nth_max}} | ||
| [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max} | ||
| export cores_per_node=${npe_node_max} | ||
| export APRUN_FV3="${launcher} -n ${npe_efcs:-0}" | ||
|
|
||
| export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} | ||
| [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max} | ||
| export APRUN_REGRID_NEMSIO="${launcher} ${LEVS:-128}" | ||
|
|
||
| elif [[ ${step} = "post" ]]; then | ||
|
|
||
| npe_node_post=${npe_node_post:-npe_node_max} | ||
| nth_max=$((npe_node_max / npe_node_post)) | ||
|
|
||
| export NTHREADS_NP=${nth_np:-1} | ||
| [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} | ||
| export APRUN_NP=${launcher} | ||
|
|
||
| export NTHREADS_DWN=${nth_dwn:-1} | ||
| [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} | ||
| export APRUN_DWN=${launcher} | ||
|
|
||
| elif [[ ${step} = "ecen" ]]; then | ||
|
|
||
| npe_node_ecen=${npe_node_ecen:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_ecen)) | ||
|
|
||
| export NTHREADS_ECEN=${nth_ecen:-${nth_max}} | ||
| [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max} | ||
| export APRUN_ECEN=${launcher} | ||
|
|
||
| export NTHREADS_CHGRES=${nth_chgres:-12} | ||
| [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max} | ||
| export APRUN_CHGRES="time" | ||
|
|
||
| export NTHREADS_CALCINC=${nth_calcinc:-1} | ||
| [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} | ||
| export APRUN_CALCINC=${launcher} | ||
|
|
||
| elif [[ ${step} = "esfc" ]]; then | ||
|
|
||
| npe_node_esfc=${npe_node_esfc:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_esfc)) | ||
|
|
||
| export NTHREADS_ESFC=${nth_esfc:-${nth_max}} | ||
| [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max} | ||
| export APRUN_ESFC="${launcher} -n ${npe_esfc:-0}" | ||
|
|
||
| export NTHREADS_CYCLE=${nth_cycle:-14} | ||
| [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} | ||
| export APRUN_CYCLE="${launcher} -n ${npe_esfc:-0}" | ||
|
|
||
| elif [[ ${step} = "epos" ]]; then | ||
|
|
||
| npe_node_epos=${npe_node_epos:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_epos)) | ||
|
|
||
| export NTHREADS_EPOS=${nth_epos:-${nth_max}} | ||
| [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max} | ||
| export APRUN_EPOS=${launcher} | ||
|
|
||
| elif [[ ${step} = "init" ]]; then | ||
|
|
||
| export APRUN=${launcher} | ||
|
|
||
| elif [[ ${step} = "postsnd" ]]; then | ||
|
|
||
| npe_node_postsnd=${npe_node_postsnd:-${npe_node_max}} | ||
| nth_max=$((npe_node_max / npe_node_postsnd)) | ||
|
|
||
| export NTHREADS_POSTSND=${nth_postsnd:-1} | ||
| [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max} | ||
| export APRUN_POSTSND=${launcher} | ||
|
|
||
| export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} | ||
| [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max} | ||
| export APRUN_POSTSNDCFP=${launcher} | ||
|
|
||
| elif [[ ${step} = "awips" ]]; then | ||
|
|
||
| echo "WARNING: ${step} is not enabled on S4!" | ||
|
|
||
| elif [[ ${step} = "gempak" ]]; then | ||
|
|
||
| echo "WARNING: ${step} is not enabled on S4!" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.