Skip to content

Commit

Permalink
GEFS Staging in exglobal_stage_ic (NOAA-EMC#1892)
Browse files Browse the repository at this point in the history
Adjusts source paths for forecast-only ICs in support of extension to ensembles.
New paths omit the resolution (that is captured by the IC id name) and allow for
a member level.

Directories with the reorganized ICs have been added to the three tier-1 machines
under glopara space.

Building on this, also adds capability to stage GEFS ICs.

Resolves NOAA-EMC#911
  • Loading branch information
AnilKumar-NOAA authored Oct 27, 2023
1 parent 77c1ff2 commit aff6ca7
Show file tree
Hide file tree
Showing 12 changed files with 181 additions and 166 deletions.
1 change: 1 addition & 0 deletions parm/config/gefs/config.base.emc.dyn
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ export FIXreg2grb2=${HOMEgfs}/fix/reg2grb2
export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops
export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops
export COMINsyn="@COMINsyn@"
export BASE_CPLIC="@BASE_CPLIC@"

# USER specific paths
export HOMEDIR="@HOMEDIR@"
Expand Down
43 changes: 0 additions & 43 deletions parm/config/gefs/config.coupled_ic

This file was deleted.

12 changes: 6 additions & 6 deletions parm/config/gefs/config.resources
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ if [[ $# -ne 1 ]]; then

echo "Must specify an input task argument to set resource variables!"
echo "argument can be any one of the following:"
echo "coupled_ic aerosol_init"
echo "stage_ic aerosol_init"
echo "sfcanl analcalc analdiag fcst post vrfy fit2obs metp arch echgres"
echo "ecen esfc efcs epos earc"
echo "init_chem mom6ic ocnpost"
Expand Down Expand Up @@ -416,12 +416,12 @@ elif [[ ${step} = "arch" || ${step} = "earc" ]]; then
eval "export memory_${step}=50GB"
fi

elif [[ ${step} = "coupled_ic" ]]; then
elif [[ ${step} = "stage_ic" ]]; then

export wtime_coupled_ic="00:15:00"
export npe_coupled_ic=1
export npe_node_coupled_ic=1
export nth_coupled_ic=1
export wtime_stage_ic="00:15:00"
export npe_stage_ic=1
export npe_node_stage_ic=1
export nth_stage_ic=1
export is_exclusive=True

elif [[ ${step} = "ecen" ]]; then
Expand Down
23 changes: 23 additions & 0 deletions parm/config/gefs/config.stage_ic
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#! /usr/bin/env bash

########## config.stage_ic ##########

echo "BEGIN: config.stage_ic"

# Get task specific resources
source "${EXPDIR}/config.resources" stage_ic

case "${CASE}" in
"C48")
export CPL_ATMIC="gefs_test"
export CPL_ICEIC="gefs_test"
export CPL_OCNIC="gefs_test"
export CPL_WAVIC="gefs_test"
;;
*)
echo "FATAL ERROR Unrecognized resolution: ${CASE}"
exit 1
;;
esac

echo "END: config.stage_ic"
24 changes: 12 additions & 12 deletions parm/config/gfs/config.stage_ic
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,22 @@ source "${EXPDIR}/config.resources" stage_ic

case "${CASE}" in
"C48" | "C96")
export CPL_ATMIC=workflowtest
export CPL_ICEIC=workflowtest
export CPL_OCNIC=workflowtest
export CPL_WAVIC=workflowtest
export CPL_ATMIC="workflow_${CASE}_refactored"
export CPL_ICEIC="workflow_${CASE}_refactored"
export CPL_OCNIC="workflow_${CASE}_refactored"
export CPL_WAVIC="workflow_${CASE}_refactored"
;;
"C384")
export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c
export CPL_ICEIC=CPC
export CPL_OCNIC=CPC3Dvar
export CPL_WAVIC=GEFSwave20210528v2
export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored
export CPL_ICEIC=CPC_refactored
export CPL_OCNIC=CPC3Dvar_refactored
export CPL_WAVIC=GEFSwave20210528v2_refactored
;;
"C768")
export CPL_ATMIC=HR2
export CPL_ICEIC=HR1
export CPL_OCNIC=HR1
export CPL_WAVIC=HR1
export CPL_ATMIC=HR2_refactored
export CPL_ICEIC=HR1_refactored
export CPL_OCNIC=HR1_refactored
export CPL_WAVIC=HR1_refactored
;;
*)
echo "FATAL ERROR Unrecognized resolution: ${CASE}"
Expand Down
148 changes: 69 additions & 79 deletions scripts/exglobal_stage_ic.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /usr/bin/env bash
#!/usr/bin/env bash

source "${HOMEgfs}/ush/preamble.sh"

Expand All @@ -8,99 +8,89 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H)
gPDY="${GDATE:0:8}"
gcyc="${GDATE:8:2}"

MEMDIR_ARRAY=()
if [[ "${RUN}" == "gefs" ]]; then
# Populate the member_dirs array based on the value of NMEM_ENS
for ((ii = 0; ii <= "${NMEM_ENS}"; ii++)); do
MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")")
done
else
MEMDIR_ARRAY+=("")
fi

# Initialize return code
err=0

error_message(){
echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})"
error_message() {
echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})"
}

###############################################################
# Start staging

# Stage the FV3 initial conditions to ROTDIR (cold start)
YMD=${PDY} HH=${cyc} generate_com -r COM_ATMOS_INPUT
[[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/gfs_ctrl.nc"
target="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
${NCP} "${source}" "${target}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
for ftype in gfs_data sfc_data; do
for tt in $(seq 1 6); do
source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/${ftype}.tile${tt}.nc"
target="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
${NCP} "${source}" "${target}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
done
done

# Stage ocean initial conditions to ROTDIR (warm start)
if [[ "${DO_OCN:-}" = "YES" ]]; then
YMD=${gPDY} HH=${gcyc} generate_com -r COM_OCEAN_RESTART
[[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}"
source="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/ocn/${OCNRES}/MOM.res.nc"
target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
${NCP} "${source}" "${target}"
for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
# Stage the FV3 initial conditions to ROTDIR (cold start)
YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT
[[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc"
tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
${NCP} "${src}" "${tgt}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
err=$((err + rc))
case "${OCNRES}" in
"500" | "100") # Only 5 degree or 1 degree ocean does not have MOM.res_[1-4].nc files
;;
"025") # Only 1/4 degree ocean has MOM.res_[1-4].nc files
for nn in $(seq 1 4); do
source="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/ocn/${OCNRES}/MOM.res_${nn}.nc"
if [[ -f "${source}" ]]; then
target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
${NCP} "${source}" "${target}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
fi
done
;;
*)
echo "FATAL ERROR: Unsupported ocean resolution ${OCNRES}"
rc=1
for ftype in gfs_data sfc_data; do
for ((tt = 1; tt <= 6; tt++)); do
src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc"
tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
${NCP} "${src}" "${tgt}"
rc=$?
tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
${NCP} "${src}" "${tgt}"
rc=$?
((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
err=$((err + rc))
;;
esac
fi

# Stage ice initial conditions to ROTDIR (warm start)
if [[ "${DO_ICE:-}" = "YES" ]]; then
YMD=${gPDY} HH=${gcyc} generate_com -r COM_ICE_RESTART
[[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}"
ICERESdec=$(echo "${ICERES}" | awk '{printf "%0.2f", $1/100}')
source="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/ice/${ICERES}/cice5_model_${ICERESdec}.res_${PDY}${cyc}.nc"
target="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
${NCP} "${source}" "${target}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
err=$((err + rc))
fi
done
done

# Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc)
if [[ "${DO_WAVE:-}" = "YES" ]]; then
YMD=${PDY} HH=${cyc} generate_com -r COM_WAVE_RESTART
[[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust
source="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/wav/${grdID}/${PDY}.${cyc}0000.restart.${grdID}"
target="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
${NCP} "${source}" "${target}"
# Stage ocean initial conditions to ROTDIR (warm start)
if [[ "${DO_OCN:-}" = "YES" ]]; then
YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART
[[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}"
src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc"
tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
${NCP} "${src}" "${tgt}"
rc=$?
(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
err=$((err + rc))
done
fi
fi
# Stage ice initial conditions to ROTDIR (warm start)
if [[ "${DO_ICE:-}" = "YES" ]]; then
YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART
[[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}"
src="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc"
tgt="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
${NCP} "${src}" "${tgt}"
rc=$?
((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
err=$((err + rc))
fi

# Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc)
if [[ "${DO_WAVE:-}" = "YES" ]]; then
YMD=${PDY} HH=${cyc} generate_com COM_WAVE_RESTART
[[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust
src="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}"
tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
${NCP} "${src}" "${tgt}"
rc=$?
((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
err=$((err + rc))
done
fi

done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
###############################################################
# Check for errors and exit if any of the above failed
if [[ "${err}" -ne 0 ]] ; then
if [[ "${err}" -ne 0 ]]; then
echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!"
exit "${err}"
fi
Expand Down
4 changes: 2 additions & 2 deletions workflow/applications/gefs.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def _get_app_configs(self):
"""
Returns the config_files that are involved in gefs
"""
configs = ['fcst']
configs = ['stage_ic', 'fcst']

if self.nens > 0:
configs += ['efcs']
Expand All @@ -32,7 +32,7 @@ def _update_base(base_in):

def get_task_names(self):

tasks = ['fcst']
tasks = ['stage_ic', 'fcst']

if self.nens > 0:
tasks += ['efcs']
Expand Down
2 changes: 1 addition & 1 deletion workflow/hosts/hera.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git'
DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump'
BASE_CPLIC: '/scratch1/NCEPDEV/climate/role.ufscpara/IC'
BASE_CPLIC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR/prototype_ICs'
PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara'
COMROOT: '/scratch1/NCEPDEV/global/glopara/com'
COMINsyn: '${COMROOT}/gfs/prod/syndat'
Expand Down
2 changes: 1 addition & 1 deletion workflow/hosts/wcoss2.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git'
DMPDIR: '/lfs/h2/emc/dump/noscrub/dump'
BASE_CPLIC: '/lfs/h2/emc/couple/noscrub/Jiande.Wang/IC'
BASE_CPLIC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/prototype_ICs'
PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}'
COMROOT: '${COMROOT:-"/lfs/h1/ops/prod/com"}'
COMINsyn: '${COMROOT}/gfs/v16.3/syndat'
Expand Down
Loading

0 comments on commit aff6ca7

Please sign in to comment.