diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5fa5dc3758..3d6531877d 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -107,6 +107,7 @@ case $MACHINE in ulimit -a APRUN="srun" LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + OMP_NUM_THREADS=4 ;; # "JET") @@ -114,6 +115,7 @@ case $MACHINE in ulimit -a APRUN="srun" LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + OMP_NUM_THREADS=4 ;; # "ODIN") @@ -427,6 +429,7 @@ fi # create_model_config_file \ cdate="$cdate" \ + nthreads=${OMP_NUM_THREADS:-1} \ run_dir="${run_dir}" || print_err_msg_exit "\ Call to function to create a model configuration file for the current cycle's (cdate) run directory (run_dir) failed: @@ -456,8 +459,9 @@ fi #----------------------------------------------------------------------- # export KMP_AFFINITY=scatter -export OMP_NUM_THREADS=1 #Needs to be 1 for dynamic build of CCPP with GFDL fast physics, was 2 before. +export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} #Needs to be 1 for dynamic build of CCPP with GFDL fast physics, was 2 before. export OMP_STACKSIZE=1024m + # #----------------------------------------------------------------------- # diff --git a/tests/baseline_configs/config.new_JPgrid.sh b/tests/baseline_configs/config.new_JPgrid.sh index d76238fe1b..c1fc566cd3 100644 --- a/tests/baseline_configs/config.new_JPgrid.sh +++ b/tests/baseline_configs/config.new_JPgrid.sh @@ -52,8 +52,8 @@ JPgrid_WIDE_HALO_WIDTH=6 DT_ATMOS="40" LAYOUT_X="8" -LAYOUT_Y="6" -BLOCKSIZE="26" +LAYOUT_Y="12" +BLOCKSIZE="13" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/ush/create_model_config_file.sh b/ush/create_model_config_file.sh index 5192bf8867..84505566db 100644 --- a/ush/create_model_config_file.sh +++ b/ush/create_model_config_file.sh @@ -48,6 +48,7 @@ function create_model_config_file() { local valid_args=( cdate \ run_dir \ +nthreads \ ) process_args valid_args "$@" # @@ -115,6 +116,7 @@ cycle directory..." set_file_param "${model_config_fp}" "ncores_per_node" "${NCORES_PER_NODE}" set_file_param "${model_config_fp}" "quilting" "${dot_quilting_dot}" set_file_param "${model_config_fp}" "print_esmf" "${dot_print_esmf_dot}" + set_file_param "${model_config_fp}" "atmos_nthreads" "${nthreads:-1}" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3SAR_wflow.sh b/ush/generate_FV3SAR_wflow.sh index 18d9d0f81b..e09a87bde3 100755 --- a/ush/generate_FV3SAR_wflow.sh +++ b/ush/generate_FV3SAR_wflow.sh @@ -113,6 +113,7 @@ settings="\ 'queue_hpss_tag': ${QUEUE_HPSS_TAG} 'queue_fcst': ${QUEUE_FCST} 'queue_fcst_tag': ${QUEUE_FCST_TAG} + 'machine': ${MACHINE} # # Workflow task names. # @@ -138,6 +139,12 @@ settings="\ 'nnodes_run_fcst': ${NNODES_RUN_FCST} 'nnodes_run_post': ${NNODES_RUN_POST} # +# Number of cores used for a task +# + 'ncores_run_fcst': ${PE_MEMBER01} + 'native_run_fcst': --cpus-per-task 4 --exclusive + 'partition_run_fcst': sjet,vjet,kjet,xjet +# # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 0eb0636f96..55a162cd63 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -347,9 +347,9 @@ predefined domain: DT_ATMOS="300" - LAYOUT_X="2" + LAYOUT_X="20" LAYOUT_Y="2" - BLOCKSIZE="2" + BLOCKSIZE="10" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -460,9 +460,9 @@ predefined domain: DT_ATMOS="40" - LAYOUT_X="36" - LAYOUT_Y="24" - BLOCKSIZE="26" + LAYOUT_X="18" + LAYOUT_Y="12" + BLOCKSIZE="46" QUILTING="TRUE" @@ -497,9 +497,9 @@ predefined domain: DT_ATMOS="40" - LAYOUT_X="34" - LAYOUT_Y="24" - BLOCKSIZE="34" + LAYOUT_X="18" + LAYOUT_Y="12" + BLOCKSIZE="46" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index d0d9d22997..1fcb15aef8 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -116,7 +116,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_GRID_TN;" "&JOBSDIR;/JREGIONAL_MAKE_GRID" @@ -136,7 +136,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_OROG_TN;" "&JOBSDIR;/JREGIONAL_MAKE_OROG" @@ -164,7 +164,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_SFC_CLIMO_TN;" "&JOBSDIR;/JREGIONAL_MAKE_SFC_CLIMO" @@ -198,7 +198,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_ICS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" @@ -220,7 +220,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_LBCS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" @@ -252,7 +252,7 @@ MODULES_RUN_TASK_FP script. {%- endfor %} {%- endif %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_ICS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_ICS" @@ -294,7 +294,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_LBCS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_LBCS" @@ -336,13 +336,21 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_FCST; &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" + {% if machine in ["JET", "HERA"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} + {% if machine == "JET" %} + {{ partition_run_fcst }} + {% endif %} + {% else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} - {{ wtime_run_fcst }} &NCORES_PER_NODE; + {% endif %} + {{ wtime_run_fcst }} &RUN_FCST_TN;{{ uscore_ensmem_name }} &LOGDIR;/&RUN_FCST_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -369,7 +377,7 @@ MODULES_RUN_TASK_FP script. {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" @@ -388,10 +396,13 @@ MODULES_RUN_TASK_FP script. fhr#fhr# - - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf0#fhr#.nc - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf0#fhr#.nc - + + + + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf0#fhr#.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf0#fhr#.nc + + diff --git a/ush/templates/model_configure b/ush/templates/model_configure index de21856353..0830165714 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_CPT_v0 b/ush/templates/model_configure.FV3_CPT_v0 index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_CPT_v0 +++ b/ush/templates/model_configure.FV3_CPT_v0 @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GFS_2017_gfdlmp b/ush/templates/model_configure.FV3_GFS_2017_gfdlmp index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GFS_2017_gfdlmp +++ b/ush/templates/model_configure.FV3_GFS_2017_gfdlmp @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GFS_2017_gfdlmp_regional b/ush/templates/model_configure.FV3_GFS_2017_gfdlmp_regional index b258980432..baa69d8790 100644 --- a/ush/templates/model_configure.FV3_GFS_2017_gfdlmp_regional +++ b/ush/templates/model_configure.FV3_GFS_2017_gfdlmp_regional @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GFS_v15p2 b/ush/templates/model_configure.FV3_GFS_v15p2 index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GFS_v15p2 +++ b/ush/templates/model_configure.FV3_GFS_v15p2 @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GFS_v16beta b/ush/templates/model_configure.FV3_GFS_v16beta index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GFS_v16beta +++ b/ush/templates/model_configure.FV3_GFS_v16beta @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GSD_SAR b/ush/templates/model_configure.FV3_GSD_SAR index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GSD_SAR +++ b/ush/templates/model_configure.FV3_GSD_SAR @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GSD_SAR_v1 b/ush/templates/model_configure.FV3_GSD_SAR_v1 index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GSD_SAR_v1 +++ b/ush/templates/model_configure.FV3_GSD_SAR_v1 @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_GSD_v0 b/ush/templates/model_configure.FV3_GSD_v0 index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_GSD_v0 +++ b/ush/templates/model_configure.FV3_GSD_v0 @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true. diff --git a/ush/templates/model_configure.FV3_RRFS_v0 b/ush/templates/model_configure.FV3_RRFS_v0 index de21856353..0830165714 100644 --- a/ush/templates/model_configure.FV3_RRFS_v0 +++ b/ush/templates/model_configure.FV3_RRFS_v0 @@ -13,7 +13,7 @@ dt_atmos: cpl: .false. calendar: 'julian' memuse_verbose: .false. -atmos_nthreads: 2 +atmos_nthreads: use_hyper_thread: .false. ncores_per_node: debug_affinity: .true.