Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history 
commit 74bf12ddcb351552372413b8f49b1d2f570ce531
Author: fanglin.yang <fanglin.yang@noaa.gov>
Date:   Tue Nov 14 21:05:49 2017 +0000

    revert all resources specification in config.fv3 to original values

    Change-Id: Ib22cb145627fab7ff93f1d1a48d69e63483805ec

commit 93783d9b056c8e6dde9836bab4b5589ce20cd79c
Author: fanglin.yang <fanglin.yang@noaa.gov>
Date:   Tue Nov 14 14:22:50 2017 -0500

    add LD_BIND_NOW=1 to running environment on Theia after maintenance to avoid esmf failure due to memroy issue

commit d48e606901d401d0406bf11e7753e26ae789fa04
Author: Rahul Mahajan <rahul.mahajan@noaa.gov>
Date:   Thu Nov 9 10:33:24 2017 -0500

    following Theia maintenance, currently the model is requiring twice as much as processors. Increase the layout_x for all resolutions. This should be reverted once the issue with Theia is resolved. Variable FCSTMODSDIR is added to config.fcst that points to the modulefiles for the model used during compilation. exglobal_fcst_nemsfv3gfs.sh is updated to load this modulefile for the appropriate machine so that the runtime modules are consistent with modules used during compilation. Also, echo out namelists right before execution to ease debugging for model configuration

commit aef7732c30979fb0b7556a3cde70cfd362f590fc
Author: fanglin.yang <fanglin.yang@noaa.gov>
Date:   Wed Nov 8 19:31:12 2017 +0000

    change WRITE_NEMSIFILE to OUTPUT_FILE after model update on 20171107, and esmf lib on THeia update maintence

    Change-Id: I17777518dc023f2a3547757751848c2ac2ab5f64

Change-Id: I81345310357b3ec9f19b677b64a537bf4e9a56eb
  • Loading branch information
@yangfanglin
yangfanglin committed Nov 14, 2017
1 parent 1014331 commit 41a6c5b
Show file tree
Hide file tree
Showing 7 changed files with 35 additions and 29 deletions.
4 changes: 2 additions & 2 deletions gfs_workflow.v15.0.0/config/config.base
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,10 +210,10 @@ export FHOUT_HF_GFS=1

# I/O QUILTING, true--use Write Component; false--use GFDL FMS
# if quilting=true, chose OUTPUT_GRID as cubed_sphere_grid in netcdf or gaussian_grid
# if gaussian_grid, set WRITE_NEMSIOFILE for nemsio or netcdf
# if gaussian_grid, set OUTPUT_FILE for nemsio or netcdf
export QUILTING=".false."
export OUTPUT_GRID="gaussian_grid"
export WRITE_NEMSIOFILE=".true."
export OUTPUT_FILE="nemsio"


# Shared parameters
Expand Down
1 change: 1 addition & 0 deletions gfs_workflow.v15.0.0/config/config.fcst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ if [ $DONST = "YES" ]; then
fi

export FORECASTSH="$BASE_GSM/scripts/exglobal_fcst_nemsfv3gfs.sh"
export FCSTMODSDIR="$BASE_NEMSfv3gfs/modulefiles"
export FCSTEXECDIR="$BASE_NEMSfv3gfs/NEMS/exe"
export FCSTEXEC="fv3_gfs_nh.prod.32bit.x"
export npe_fv3=$npe_fcst # This is model resolution dependent, see note above
Expand Down
1 change: 1 addition & 0 deletions gfs_workflow.v15.0.0/env/THEIA.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export MPI_MEMMAP_OFF=1
export MP_STDOUTMODE="ORDERED"
export OMP_STACKSIZE=2048000
export NTHSTACK=1024000000
export LD_BIND_NOW=1

if [ $step = "prep" -o $step = "prepbufr" ]; then

Expand Down
2 changes: 1 addition & 1 deletion gfs_workflow.v15.0.0/jobs/fcst.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ elif [ $CDUMP = "gfs" ]; then
[[ $status -ne 0 ]] && exit $status
fi

if [ $WRITE_NEMSIOFILE = ".false." -o $QUILTING = ".false." ]; then
if [ $OUTPUT_FILE != "nemsio" -o $QUILTING = ".false." ]; then
# Convert NetCDF to nemsio
$NC2NEMSIOSH
status=$?
Expand Down
4 changes: 2 additions & 2 deletions global_shared.v15.0.0/parm/parm_fv3diag/diag_table2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
"gfs_dyn", "graupel", "graupel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "w", "vvel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "w", "vvel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2

Expand Down
4 changes: 2 additions & 2 deletions global_shared.v15.0.0/parm/parm_fv3diag/diag_table2_da
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
"gfs_dyn", "graupel", "graupel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2
#"gfs_dyn", "w", "vvel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "w", "vvel", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2
"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2

Expand Down
48 changes: 26 additions & 22 deletions global_shared.v15.0.0/scripts/exglobal_fcst_nemsfv3gfs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,8 @@ if [ $VERBOSE = "YES" ] ; then
set -x
fi

# This should be in the script that calls this script, not here
machine=${machine:-"WCOSS_C"}
machine=$(echo $machine | tr '[a-z]' '[A-Z]')
if [ $machine = "WCOSS_C" ] ; then
. $MODULESHOME/init/sh 2>/dev/null
PRGENV=${PRGENV:-intel}
HUGEPAGES=${HUGEPAGES:-hugepages4M}
module unload prod_util iobuf PrgEnv-$PRGENV craype-$HUGEPAGES 2>/dev/null
module load prod_util iobuf PrgEnv-$PRGENV craype-$HUGEPAGES 2>/dev/null
module load intel/16.3.210 2>/dev/null
module use /usrx/local/dev/modulefiles
export IOBUF_PARAMS=${IOBUF_PARAMS:-'*:size=8M:verbose'}
export MPICH_GNI_COLL_OPT_OFF=${MPICH_GNI_COLL_OPT_OFF:-MPI_Alltoallv}
export MKL_CBWR=AVX2
module use /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles 2>/dev/null
module load esmf/7.1.0bs34 2>/dev/null
elif [ $machine = "THEIA" ]; then
. $MODULESHOME/init/sh 2>/dev/null
module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles 2>/dev/null
module load esmf/7.1.0bs34 2>/dev/null
fi

# Cycling and forecast hour specific parameters
CASE=${CASE:-C768}
Expand Down Expand Up @@ -98,7 +79,8 @@ MEMBER=${MEMBER:-"-1"} # -1: control, 0: ensemble mean, >0: ensemble member $MEM
ENS_NUM=${ENS_NUM:-1} # Single executable runs multiple members (e.g. GEFS)

# Model specific stuff
FCSTEXECDIR=${FCSTEXECDIR:-$BASE_GSM/sorc/fv3gfs.fd/BUILD/bin}
FCSTMODSDIR=${FCSTMODSDIR:-$BASE_GSM/sorc/fv3gfs.fd/modulefiles}
FCSTEXECDIR=${FCSTEXECDIR:-$BASE_GSM/sorc/fv3gfs.fd/NEMS/exe}
FCSTEXEC=${FCSTEXEC:-fv3_gfs.x}
PARM_FV3DIAG=${PARM_FV3DIAG:-$BASE_GSM/parm/parm_fv3diag}

Expand All @@ -114,7 +96,7 @@ MONO=${MONO:-"non-mono"} # choices: mono, non-mono

QUILTING=${QUILTING:-".true."}
OUTPUT_GRID=${OUTPUT_GRID:-"gaussian_grid"}
WRITE_NEMSIOFILE=${WRITE_NEMSIOFILE:-".true."}
OUTPUT_FILE=${OUTPUT_FILE:-"nemsio"}
WRITE_NEMSIOFLIP=${WRITE_NEMSIOFLIP:-".true."}

rCDUMP=${rCDUMP:-$CDUMP}
Expand Down Expand Up @@ -495,7 +477,7 @@ write_tasks_per_group: ${WRTTASK_PER_GROUP:-24}
num_files: ${NUM_FILES:-2}
filename_base: '${CDUMP}.t${chh}z.atm' '${CDUMP}.t${chh}z.sfc'
output_grid: $OUTPUT_GRID
write_nemsiofile: $WRITE_NEMSIOFILE
output_file: $OUTPUT_FILE
write_nemsioflip: $WRITE_NEMSIOFLIP
imo: $LONB
jmo: $LATB
Expand Down Expand Up @@ -847,8 +829,30 @@ EOF

fi

# echo out the final namelists
cat nems.configure
cat model_configure
cat input.nml

#------------------------------------------------------------------
# setup the runtime environment and run the executable
if [ $machine = "WCOSS_C" ] ; then
HUGEPAGES=${HUGEPAGES:-hugepages4M}
. $MODULESHOME/init/sh 2>/dev/null
module purge 2>/dev/null
module use $FCSTMODSDIR/wcoss_cray 2>/dev/null
module load fv3 2>/dev/null
module load prod_util iobuf craype-$HUGEPAGES 2>/dev/null
export IOBUF_PARAMS=${IOBUF_PARAMS:-'*:size=8M:verbose'}
export MPICH_GNI_COLL_OPT_OFF=${MPICH_GNI_COLL_OPT_OFF:-MPI_Alltoallv}
export MKL_CBWR=AVX2
elif [ $machine = "THEIA" ]; then
. $MODULESHOME/init/sh 2>/dev/null
module purge 2>/dev/null
module use $FCSTMODSDIR/theia 2>/dev/null
module load fv3 2>/dev/null
fi

cd $DATA
$NCP $FCSTEXECDIR/$FCSTEXEC $DATA/.
export OMP_NUM_THREADS=$NTHREADS_FV3
Expand Down

0 comments on commit 41a6c5b

Please sign in to comment.