Skip to content

Commit

Permalink
[develop] Split calling of METplus's Pb2nc tool into separate task (f…
Browse files Browse the repository at this point in the history
…rom PointStat and EnsembleStat tasks) (#668)

Split calls to METplus's Pb2nc tool out of the tasks that run:

 1. deterministic verification (vx) using the PointStat tool,
 2. ensemble vx using the EnsembleStat, and
 3. verification on the ensemble mean and ensemble probabilities using the PointStat tool

into separate tasks. This new task is run for each cycle to convert NDAS prep-bufr files to NetCDF format. Also, include additional cleanup of vx tasks.
  • Loading branch information
gsketefian authored Mar 20, 2023
1 parent d917ace commit f8b4eb4
Show file tree
Hide file tree
Showing 46 changed files with 1,573 additions and 1,140 deletions.
85 changes: 85 additions & 0 deletions jobs/JREGIONAL_RUN_MET_PB2NC_OBS
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/bin/bash

#
#-----------------------------------------------------------------------
#
#
#
#-----------------------------------------------------------------------
#

#
#-----------------------------------------------------------------------
#
# Source the variable definitions file and the bash utility functions.
#
#-----------------------------------------------------------------------
#
. $USHdir/source_util_funcs.sh
source_config_for_task "task_run_met_pb2nc_obs" ${GLOBAL_VAR_DEFNS_FP}
. $USHdir/job_preamble.sh
#
#-----------------------------------------------------------------------
#
# Save current shell options (in a global array). Then set new options
# for this script/function.
#
#-----------------------------------------------------------------------
#
{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
#
#-----------------------------------------------------------------------
#
# Get the full path to the file in which this script/function is located
# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in
# which the file is located (scrfunc_dir).
#
#-----------------------------------------------------------------------
#
scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" )
scrfunc_fn=$( basename "${scrfunc_fp}" )
scrfunc_dir=$( dirname "${scrfunc_fp}" )
#
#-----------------------------------------------------------------------
#
# Print message indicating entry into script.
#
#-----------------------------------------------------------------------
#
print_info_msg "
========================================================================
Entering script: \"${scrfunc_fn}\"
In directory: \"${scrfunc_dir}\"
This is the J-job script for the task that runs METplus for point-stat
by initialization time for all forecast hours.
========================================================================"
#
#-----------------------------------------------------------------------
#
# Call the ex-script for this J-job and pass to it the necessary varia-
# bles.
#
#-----------------------------------------------------------------------
#
$SCRIPTSdir/exregional_run_met_pb2nc_obs.sh || \
print_err_msg_exit "\
Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
#
#-----------------------------------------------------------------------
#
# Run job postamble.
#
#-----------------------------------------------------------------------
#
job_postamble
#
#-----------------------------------------------------------------------
#
# Restore the shell options saved at the beginning of this script/func-
# tion.
#
#-----------------------------------------------------------------------
#
{ restore_shell_opts; } > /dev/null 2>&1

146 changes: 112 additions & 34 deletions parm/FV3LAM_wflow.xml

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions parm/metplus/EnsembleStat_APCP01h.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Ensemble-Stat METplus Configuration
# EnsembleStat METplus Configuration

[config]

Expand Down Expand Up @@ -37,8 +37,8 @@ OBTYPE = CCPA

#ENSEMBLE_STAT_DESC = # not in other file

# The MET ensemble_stat logging level
# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default.
# The METplus EnsembleStat logging level
# 0 quiet to 5 loud, Verbosity setting for METplus EnsembleStat output, 2 is default.
# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf
#LOG_ENSEMBLE_STAT_VERBOSITY = 2

Expand Down Expand Up @@ -189,13 +189,13 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS}
INPUT_BASE = {ENV[INPUT_BASE]}
OUTPUT_BASE = {ENV[OUTPUT_BASE]}

# Forecast model input directory for ensemble_stat
# Forecast model input directory for EnsembleStat
FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE}

# Point observation input dir for ensemble_stat
# Point observation input dir for EnsembleStat
OBS_ENSEMBLE_STAT_POINT_INPUT_DIR =

# Grid observation input dir for ensemble_stat
# Grid observation input dir for EnsembleStat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]}

# directory containing climatology mean input to EnsembleStat
Expand All @@ -206,7 +206,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR =
# Not used in this example
ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR =

# output directory for ensemble_stat
# output directory for EnsembleStat
ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}

# directory containing log files
Expand Down Expand Up @@ -241,11 +241,11 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE =
# Not used in this example
ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE =

ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/EnsembleStat

# Specify the name of the metplus.log file
LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}

# Specify where the location and name of the final metplus_final.conf
METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_01h.conf
METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/EnsembleStat/metplus_final.APCP_01h.conf

24 changes: 12 additions & 12 deletions parm/metplus/EnsembleStat_APCP03h.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Ensemble-Stat METplus Configuration
# EnsembleStat METplus Configuration

[config]

Expand Down Expand Up @@ -48,8 +48,8 @@ FCST_IS_PROB = false

#ENSEMBLE_STAT_DESC = # not in other file

# The MET ensemble_stat logging level
# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default.
# The MET EnsembleStat logging level
# 0 quiet to 5 loud, Verbosity setting for MET EnsembleStat output, 2 is default.
# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf
#LOG_ENSEMBLE_STAT_VERBOSITY = 2

Expand All @@ -72,7 +72,7 @@ ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0

ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_03h_{OBTYPE}

# MET Configuration file for ensemble_stat
# MET Configuration file for EnsembleStat
ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped

# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required.
Expand Down Expand Up @@ -199,16 +199,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS}
INPUT_BASE = {ENV[INPUT_BASE]}
OUTPUT_BASE = {ENV[OUTPUT_BASE]}
#
# Forecast model input directory for ensemble_stat
# Forecast model input directory for EnsembleStat
#
FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[MEM_BASE]}
#
# Point observation input dir for ensemble_stat
# Point observation input dir for EnsembleStat
#
OBS_ENSEMBLE_STAT_POINT_INPUT_DIR =

# Grid observation input dir for ensemble_stat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine_obs
# Grid observation input dir for EnsembleStat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/PcpCombine_obs

# directory containing climatology mean input to EnsembleStat
# Not used in this example
Expand All @@ -218,7 +218,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR =
# Not used in this example
ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR =

# output directory for ensemble_stat
# output directory for EnsembleStat
ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}
#
# Directory for staging data.
Expand All @@ -233,7 +233,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03
# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to
# FCST_ENSEMBLE_STAT_INPUT_DIR.
#
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h.nc
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/PcpCombine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h.nc

# Template to look for point observations.
# Example precip24_2010010112.nc
Expand All @@ -258,12 +258,12 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE =
# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to
# ENSEMBLE_STAT_OUTPUT_DIR.
#
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/EnsembleStat
#
# Specify the name of the METplus log file.
#
LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}
#
# Specify the location and name of the final METplus conf file.
#
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_03h.conf
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/EnsembleStat/metplus_final.APCP_03h.conf
24 changes: 12 additions & 12 deletions parm/metplus/EnsembleStat_APCP06h.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Ensemble-Stat METplus Configuration
# EnsembleStat METplus Configuration

[config]

Expand Down Expand Up @@ -48,8 +48,8 @@ FCST_IS_PROB = false

#ENSEMBLE_STAT_DESC = # not in other file

# The MET ensemble_stat logging level
# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default.
# The MET EnsembleStat logging level
# 0 quiet to 5 loud, Verbosity setting for MET EnsembleStat output, 2 is default.
# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf
#LOG_ENSEMBLE_STAT_VERBOSITY = 2

Expand All @@ -72,7 +72,7 @@ ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0

ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_06h_{OBTYPE}

# MET Configuration file for ensemble_stat
# MET Configuration file for EnsembleStat
ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped

# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required.
Expand Down Expand Up @@ -199,16 +199,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS}
INPUT_BASE = {ENV[INPUT_BASE]}
OUTPUT_BASE = {ENV[OUTPUT_BASE]}
#
# Forecast model input directory for ensemble_stat
# Forecast model input directory for EnsembleStat
#
FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[MEM_BASE]}
#
# Point observation input dir for ensemble_stat
# Point observation input dir for EnsembleStat
#
OBS_ENSEMBLE_STAT_POINT_INPUT_DIR =

# Grid observation input dir for ensemble_stat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine_obs
# Grid observation input dir for EnsembleStat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/PcpCombine_obs

# directory containing climatology mean input to EnsembleStat
# Not used in this example
Expand All @@ -218,7 +218,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR =
# Not used in this example
ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR =

# output directory for ensemble_stat
# output directory for EnsembleStat
ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}
#
# Directory for staging data.
Expand All @@ -233,7 +233,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06
# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to
# FCST_ENSEMBLE_STAT_INPUT_DIR.
#
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h.nc
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/PcpCombine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h.nc

# Template to look for point observations.
# Example precip24_2010010112.nc
Expand All @@ -258,12 +258,12 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE =
# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to
# ENSEMBLE_STAT_OUTPUT_DIR.
#
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/EnsembleStat
#
# Specify the name of the METplus log file.
#
LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}
#
# Specify the location and name of the final METplus conf file.
#
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_06h.conf
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/EnsembleStat/metplus_final.APCP_06h.conf
24 changes: 12 additions & 12 deletions parm/metplus/EnsembleStat_APCP24h.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Ensemble-Stat METplus Configuration
# EnsembleStat METplus Configuration

[config]

Expand Down Expand Up @@ -48,8 +48,8 @@ FCST_IS_PROB = false

#ENSEMBLE_STAT_DESC = # not in other file

# The MET ensemble_stat logging level
# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default.
# The MET EnsembleStat logging level
# 0 quiet to 5 loud, Verbosity setting for MET EnsembleStat output, 2 is default.
# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf
#LOG_ENSEMBLE_STAT_VERBOSITY = 2

Expand All @@ -72,7 +72,7 @@ ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0

ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_24h_{OBTYPE}

# MET Configuration file for ensemble_stat
# MET Configuration file for EnsembleStat
ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped

# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required.
Expand Down Expand Up @@ -199,16 +199,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS}
INPUT_BASE = {ENV[INPUT_BASE]}
OUTPUT_BASE = {ENV[OUTPUT_BASE]}
#
# Forecast model input directory for ensemble_stat
# Forecast model input directory for EnsembleStat
#
FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[MEM_BASE]}
#
# Point observation input dir for ensemble_stat
# Point observation input dir for EnsembleStat
#
OBS_ENSEMBLE_STAT_POINT_INPUT_DIR =

# Grid observation input dir for ensemble_stat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine_obs
# Grid observation input dir for EnsembleStat
OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/PcpCombine_obs

# directory containing climatology mean input to EnsembleStat
# Not used in this example
Expand All @@ -218,7 +218,7 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR =
# Not used in this example
ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR =

# output directory for ensemble_stat
# output directory for EnsembleStat
ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}
#
# Directory for staging data.
Expand All @@ -233,7 +233,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24
# The template FCST_ENSEMBLE_STAT_INPUT_TEMPLATE is relative to
# FCST_ENSEMBLE_STAT_INPUT_DIR.
#
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h.nc
FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/PcpCombine_fcst/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h.nc

# Template to look for point observations.
# Example precip24_2010010112.nc
Expand All @@ -258,12 +258,12 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE =
# The template ENSEMBLE_STAT_OUTPUT_TEMPLATE is relative to
# ENSEMBLE_STAT_OUTPUT_DIR.
#
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat
ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/EnsembleStat
#
# Specify the name of the METplus log file.
#
LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}
#
# Specify the location and name of the final METplus conf file.
#
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_24h.conf
METPLUS_CONF = {OUTPUT_BASE}/{ENV[CDATE]}/metprd/EnsembleStat/metplus_final.APCP_24h.conf
Loading

0 comments on commit f8b4eb4

Please sign in to comment.