Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release/public-v2: bit-for-bit reproducibility for regional runs when changing MPI decomposition #409

Merged
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 1 files
+1 −1 atmos_cubed_sphere
178 changes: 125 additions & 53 deletions tests/RegressionTests_cheyenne.gnu.log

Large diffs are not rendered by default.

200 changes: 136 additions & 64 deletions tests/RegressionTests_cheyenne.intel.log

Large diffs are not rendered by default.

178 changes: 125 additions & 53 deletions tests/RegressionTests_gaea.intel.log

Large diffs are not rendered by default.

576 changes: 0 additions & 576 deletions tests/RegressionTests_hera.gnu.log

This file was deleted.

178 changes: 125 additions & 53 deletions tests/RegressionTests_hera.intel.log

Large diffs are not rendered by default.

178 changes: 125 additions & 53 deletions tests/RegressionTests_jet.intel.log

Large diffs are not rendered by default.

2,923 changes: 344 additions & 2,579 deletions tests/RegressionTests_orion.intel.log

Large diffs are not rendered by default.

2,471 changes: 0 additions & 2,471 deletions tests/RegressionTests_wcoss_cray.log

This file was deleted.

2,482 changes: 0 additions & 2,482 deletions tests/RegressionTests_wcoss_dell_p3.log

This file was deleted.

482 changes: 0 additions & 482 deletions tests/UnitTests_hera.intel.log

This file was deleted.

15 changes: 13 additions & 2 deletions tests/rt.conf
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_RRFS_v1alpha

# Regional tests
RUN | fv3_ccpp_regional_control | standard | | fv3 |
# The following test does NOT reproduce the control baseline
#RUN | fv3_ccpp_regional_decomp | standard | | |
RUN | fv3_ccpp_regional_decomp | standard | | |
RUN | fv3_ccpp_regional_2threads | standard | | |
RUN | fv3_ccpp_regional_coldstart | standard | | |
# The following test does NOT reproduce the control baseline
#RUN | fv3_ccpp_regional_warmstart | standard | | | fv3_ccpp_regional_coldstart
RUN | fv3_ccpp_regional_v15p2 | standard | | fv3 |
RUN | fv3_ccpp_regional_v15p2_2threads | standard | | |
# This test only reproduces in REPRO mode (because of GFDL-MP fast physics in 32-bit)
#RUN | fv3_ccpp_regional_v15p2_decomp | standard | | |

# Global tests
RUN | fv3_ccpp_rrfs_v1alpha | standard | | fv3 |
Expand All @@ -23,6 +25,15 @@ RUN | fv3_ccpp_rrfs_v1alpha_coldstart
#RUN | fv3_ccpp_rrfs_v1alpha_warmstart | standard | | | fv3_ccpp_rrfs_v1alpha_coldstart
RUN | fv3_ccpp_gfs_v15p2 | standard | | fv3 |

###############################################################################################################################
# CCPP REPRO TESTS #
###############################################################################################################################

COMPILE | CCPP=Y SUITES=FV3_GFS_v15p2,FV3_RRFS_v1alpha REPRO=Y | standard | | fv3 |
# Regional tests
RUN | fv3_ccpp_regional_v15p2 | standard | | fv3 |
RUN | fv3_ccpp_regional_v15p2_decomp | standard | | |

###############################################################################################################################
# CCPP DEBUG TESTS #
###############################################################################################################################
Expand Down
11 changes: 3 additions & 8 deletions tests/rt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ elif [[ $MACHINE_ID = orion.* ]]; then
# ACCNR= # detected in detect_machine.sh
PARTITION=orion
dprefix=/work/noaa/stmp/${USER}
DISKNM=/work/noaa/fv3-cam/djovic/RT
DISKNM=/work/noaa/nems/emc.nemspara/RT
STMP=$dprefix/stmp
PTMP=$dprefix/stmp

Expand Down Expand Up @@ -356,11 +356,6 @@ SINGLE_NAME=''

TESTS_FILE='rt.conf'

if [[ $MACHINE_ID = orion.* ]]; then
TESTS_FILE='rt_orion.conf'
fi


SET_ID='standard'
while getopts ":cfsl:mn:kreh" opt; do
case $opt in
Expand Down Expand Up @@ -418,9 +413,9 @@ if [[ $SINGLE_NAME != '' ]]; then
fi

if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210126/${COMPILER^^}}
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210208/${COMPILER^^}}
else
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210126}
RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/ufs-public-release-v2-20210208}
fi

shift $((OPTIND-1))
Expand Down
7 changes: 6 additions & 1 deletion tests/tests/fv3_ccpp_regional_decomp
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ export CNTL_DIR=fv3_regional_control

export LIST_FILES="atmos_4xdaily.nc \
phyf000.nc \
phyf006.nc \
phyf012.nc \
dynf000.nc \
dynf006.nc \
dynf012.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
RESTART/fv_srf_wnd.res.tile1.nc \
RESTART/fv_tracer.res.tile1.nc \
RESTART/sfc_data.nc \
Expand Down Expand Up @@ -68,3 +69,7 @@ RUN_SCRIPT=rt_fv3.sh

# Regional option
export WRITE_RESTART_WITH_BCS=.true.

if [[ $MACHINE_ID = *.gnu ]]; then
export WLCLK=60
fi
61 changes: 61 additions & 0 deletions tests/tests/fv3_ccpp_regional_v15p2_2threads
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
###############################################################################
#
# FV3 CCPP regional control test using GFS v15p2 suite with 32-bit dynamics
#
###############################################################################

export TEST_DESCR="Compare FV3 CCPP regional GFS v15p2 results with previous trunk version"

export CNTL_DIR=fv3_regional_v15p2

export LIST_FILES="atmos_4xdaily.nc \
phyf000.nc \
phyf012.nc \
dynf000.nc \
dynf012.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
RESTART/fv_srf_wnd.res.tile1.nc \
RESTART/fv_tracer.res.tile1.nc \
RESTART/sfc_data.nc \
RESTART/phy_data.nc"

export_fv3
export THRD=2
export TASKS=80
export TPN=$(expr $TPN / 2)
export NODES=$(expr $TASKS / $TPN + 1)

export INPES=20
export JNPES=2

export FHMAX=12
export FDIAG=3

DT_ATMOS="300"

export FV3_RUN=ccpp_regional_run.IN
export CCPP_SUITE=FV3_GFS_v15p2
export CCPP_LIB_DIR=ccpp/lib
export INPUT_NML=ccpp_regional.nml.IN
export MODEL_CONFIGURE=ccpp_regional-model_configure.IN

# Ozone / stratospheric H2O
export OZ_PHYS_OLD=.F.
export OZ_PHYS_NEW=.T.
export H2O_PHYS=.T.

# GFDL microphysics
export IMP_PHYSICS=11

# Other namelist switches
export HYBEDMF=.T.
export FHCYC=0

RUN_SCRIPT=rt_fv3.sh

# Regional option
export WRITE_RESTART_WITH_BCS=.true.

export WLCLK=60
60 changes: 60 additions & 0 deletions tests/tests/fv3_ccpp_regional_v15p2_decomp
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
###############################################################################
#
# FV3 CCPP regional control test using GFS v15p2 suite with 32-bit dynamics
#
###############################################################################

export TEST_DESCR="Compare FV3 CCPP regional GFS v15p2 results with previous trunk version"

export CNTL_DIR=fv3_regional_v15p2

export LIST_FILES="atmos_4xdaily.nc \
phyf000.nc \
phyf012.nc \
dynf000.nc \
dynf012.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_srf_wnd.res.tile1.nc \
RESTART/fv_tracer.res.tile1.nc \
RESTART/sfc_data.nc \
RESTART/phy_data.nc"

export_fv3
export TASKS=80
export NODES=$(expr $TASKS / $TPN + 1)

export INPES=10
export JNPES=4

export FHMAX=12
export FDIAG=3

DT_ATMOS="300"

export FV3_RUN=ccpp_regional_run.IN
export CCPP_SUITE=FV3_GFS_v15p2
export CCPP_LIB_DIR=ccpp/lib
export INPUT_NML=ccpp_regional.nml.IN
export MODEL_CONFIGURE=ccpp_regional-model_configure.IN

# Ozone / stratospheric H2O
export OZ_PHYS_OLD=.F.
export OZ_PHYS_NEW=.T.
export H2O_PHYS=.T.

# GFDL microphysics
export IMP_PHYSICS=11

# Other namelist switches
export HYBEDMF=.T.
export FHCYC=0

RUN_SCRIPT=rt_fv3.sh

# Regional option
export WRITE_RESTART_WITH_BCS=.true.

if [[ $MACHINE_ID = *.gnu ]]; then
export WLCLK=60
fi