From 3cc6bb1328ee8987c8f9a3552dd9c8bd595c00bb Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Tue, 28 Mar 2023 14:51:08 -0600 Subject: [PATCH 01/33] Change `model_total_valid_times` attrib to int32 --- core/ioMod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/ioMod.py b/core/ioMod.py index ba9b6a2..cda4440 100755 --- a/core/ioMod.py +++ b/core/ioMod.py @@ -162,7 +162,7 @@ def output_final_ldasin(self, ConfigOptions, geoMetaWrfHydro, MpiConfig): break try: - idOut.model_total_valid_times = float(ConfigOptions.actual_output_steps) + idOut.model_total_valid_times = np.int32(ConfigOptions.actual_output_steps) except: ConfigOptions.errMsg = "Unable to create total_valid_times global attribute in: " + self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) From c61cb6d8e346a13798c0ca1d8f5a3fd588f4829b Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Fri, 31 Mar 2023 13:59:01 -0600 Subject: [PATCH 02/33] Change globalNdv to -999999 --- core/config.py | 8 ++++---- core/forecastMod.py | 4 ++-- core/ioMod.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/config.py b/core/config.py index 49c1692..515832b 100755 --- a/core/config.py +++ b/core/config.py @@ -93,7 +93,7 @@ def __init__(self, config): self.customFcstFreq = None self.rqiMethod = None self.rqiThresh = 1.0 - self.globalNdv = -9999.0 + self.globalNdv = -999999.0 self.d_program_init = datetime.datetime.utcnow() self.errFlag = 0 self.nwmVersion = None @@ -1115,7 +1115,7 @@ def read_config(self): except configparser.NoOptionError: err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section ' 'in the configuration file.') - + # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace # or new line characters. for dirTmp in range(0, len(self.supp_precip_dirs)): @@ -1123,7 +1123,7 @@ def read_config(self): if not os.path.isdir(self.supp_precip_dirs[dirTmp]): err_handler.err_out_screen('Unable to locate supp pcp directory: ' + self.supp_precip_dirs[dirTmp]) - #Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input + #Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input if 11 in self.supp_precip_forcings: if len(self.supp_precip_forcings) != 1: err_handler.err_out_screen('Alaska Stage IV/MRMS SuppPcp option is only supported as a standalone option') @@ -1295,4 +1295,4 @@ def use_data_at_current_time(self): hrs_since_start = self.current_output_date - self.current_fcst_cycle return hrs_since_start <= datetime.timedelta(hours = self.supp_pcp_max_hours) else: - return True + return True diff --git a/core/forecastMod.py b/core/forecastMod.py index 59d3d3e..91b7fab 100755 --- a/core/forecastMod.py +++ b/core/forecastMod.py @@ -123,7 +123,7 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo show_message = True for outStep in range(1, ConfigOptions.num_output_steps + 1): # Reset out final grids to missing values. - OutputObj.output_local[:, :, :] = -9999.0 + OutputObj.output_local[:, :, :] = ConfigOptions.globalNdv ConfigOptions.current_output_step = outStep OutputObj.outDate = ConfigOptions.current_fcst_cycle + datetime.timedelta( @@ -229,7 +229,7 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo if forceKey == 10: ConfigOptions.currentCustomForceNum = ConfigOptions.currentCustomForceNum + 1 - + else: # Process supplemental precipitation if we specified in the configuration file. if ConfigOptions.number_supp_pcp > 0: diff --git a/core/ioMod.py b/core/ioMod.py index cda4440..fabba8b 100755 --- a/core/ioMod.py +++ b/core/ioMod.py @@ -26,7 +26,7 @@ def __init__(self,GeoMetaWrfHydro): self.output_local = None self.outPath = None self.outDate = None - self.out_ndv = -9999 + self.out_ndv = -999999 # Create local "slabs" to hold final output grids. These # will be collected during the output routine below. From c23bb81decc15034838c6e483b1fa8aaeb150a73 Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Fri, 31 Mar 2023 13:59:38 -0600 Subject: [PATCH 03/33] Add to ignored global metadata --- core/ioMod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/ioMod.py b/core/ioMod.py index fabba8b..cbdbfa3 100755 --- a/core/ioMod.py +++ b/core/ioMod.py @@ -171,7 +171,7 @@ def output_final_ldasin(self, ConfigOptions, geoMetaWrfHydro, MpiConfig): if ConfigOptions.spatial_meta is not None: # apply spatial_global_atts to output globals attrs for k, v in geoMetaWrfHydro.spatial_global_atts.items(): - if k not in ("Source_Software", "history", "processing_notes"): # don't add these + if k not in ("Source_Software", "history", "processing_notes", "version"): # don't add these idOut.setncattr(k, v) # Create variables. From a9303092b9d5aed873459d3c5d7ec491db373b5e Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Thu, 13 Apr 2023 16:52:16 -0600 Subject: [PATCH 04/33] Don't mask NLDAS params when reading netCDF var The _FillValue in the NLDAS params needs to be explicity changed to the globalNdv value so the masked areas can be reconstructed on the other side of an MPI broadcast --- core/bias_correction.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/bias_correction.py b/core/bias_correction.py index c985811..30e8fcf 100755 --- a/core/bias_correction.py +++ b/core/bias_correction.py @@ -370,7 +370,7 @@ def ncar_sw_hrrr_bias_correct(input_forcings, geo_meta_wrf_hydro, config_options # the cosine of the sol_zen_ang is proportional to the solar intensity # (not accounting for humidity or cloud cover) sol_cos = np.sin(geo_meta_wrf_hydro.latitude_grid * d2r) * math.sin(decl) + np.cos(geo_meta_wrf_hydro.latitude_grid * d2r) * math.cos(decl) * np.cos(ha) - + # Check for any values outside of [-1,1] (this can happen due to floating point rounding) sol_cos[np.where(sol_cos < -1.0)] = -1.0 sol_cos[np.where(sol_cos > 1.0)] = 1.0 @@ -908,6 +908,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for # Open the NetCDF file. try: id_nldas_param = Dataset(nldas_param_file, 'r') + id_nldas_param.set_auto_mask(False) # don't mask missing since it won't survive broadcast except OSError as err: config_options.errMsg = "Unable to open parameter file: " + nldas_param_file + " (" + str(err) + ")" err_handler.log_critical(config_options, mpi_config) @@ -1236,7 +1237,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for # regridding object. # 6.) Place the data into the final output arrays for further processing (downscaling). # 7.) Reset variables for memory efficiency and exit the routine. - + np.seterr(all='ignore') if mpi_config.rank == 0: From 06e460d948a718b0cfbecebe31ed7ac27e1957ae Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Tue, 15 Aug 2023 22:20:35 -0600 Subject: [PATCH 05/33] Fix issues with MRMS overlaying HRRR/RAP * Re-include removed HRRR precipitaion for use in Short-Range forecasts * Fix issue with MRMS missing data when used as an overlay in AnA --- core/forcingInputMod.py | 8 ++++---- core/regrid.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/core/forcingInputMod.py b/core/forcingInputMod.py index 6973775..a800b99 100755 --- a/core/forcingInputMod.py +++ b/core/forcingInputMod.py @@ -204,7 +204,7 @@ def define_product(self): 3: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', 'DLWRF', 'PRES'], 4: None, - 5: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'DSWRF', + 5: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', 'DLWRF', 'PRES'], 6: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', 'DLWRF', 'PRES'], @@ -245,7 +245,7 @@ def define_product(self): 4: None, 5: ['2 m above ground', '2 m above ground', '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface'], + 'surface', 'surface', 'surface', 'surface'], 6: ['2 m above ground', '2 m above ground', '10 m above ground', '10 m above ground', 'surface', 'surface', 'surface', 'surface'], @@ -296,7 +296,7 @@ def define_product(self): 4: None, 5: ['TMP_2maboveground', 'SPFH_2maboveground', 'UGRD_10maboveground', 'VGRD_10maboveground', - 'DSWRF_surface', 'DLWRF_surface', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', 'PRES_surface'], 6: ['TMP_2maboveground', 'SPFH_2maboveground', 'UGRD_10maboveground', 'VGRD_10maboveground', @@ -380,7 +380,7 @@ def define_product(self): 2: None, 3: [4,5,0,1,3,7,2,6], 4: None, - 5: [4,5,0,1,7,2,6], + 5: [4,5,0,1,3,7,2,6], 6: [4,5,0,1,3,7,2,6], 7: [4,5,0,1,3,7,2,6], 8: [4,5,0,1,3,6], diff --git a/core/regrid.py b/core/regrid.py index f3224cc..de977a1 100755 --- a/core/regrid.py +++ b/core/regrid.py @@ -83,7 +83,7 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co except: config_options.errMsg = f"Unable to open input NetCDF file: {input_forcings.file_in}" err_handler.log_critical(config_options, mpi_config) - + ds.set_auto_scale(True) ds.set_auto_mask(False) @@ -126,7 +126,7 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co np.float32) input_forcings.regridded_forcings2 = np.empty([8, wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], np.float32) - + for force_count, nc_var in enumerate(input_forcings.netcdf_var_names): var_tmp = None if mpi_config.rank == 0: @@ -139,7 +139,7 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = f"Unable to extract: {nc_var} from: {input_forcings.file_in2} ({str(err)})" err_handler.log_critical(config_options, mpi_config) - + err_handler.check_program_status(config_options, mpi_config) var_sub_tmp = mpi_config.scatter_array(input_forcings, var_tmp, config_options) err_handler.check_program_status(config_options, mpi_config) @@ -190,7 +190,7 @@ def _regrid_ak_ext_ana_pcp_stage4(supplemental_precip, config_options, wrf_hydro lat_var = "latitude" lon_var = "longitude" - + if supplemental_precip.fileType != NETCDF: # This file shouldn't exist.... but if it does (previously failed # execution of the program), remove it..... @@ -2044,9 +2044,13 @@ def regrid_mrms_hourly(supplemental_precip, config_options, wrf_hydro_geo_meta, # Set any pixel cells outside the input domain to the global missing value, and set negative precip values to 0 try: + if len(np.argwhere(supplemental_precip.esmf_field_out.data < 0)) > 0: + supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = 0 + config_options.statusMsg = "WARNING: Found negative precipitation values in MRMS data, setting to 0" + err_handler.log_warning(config_options, mpi_config) + supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.regridded_mask == 0)] = \ config_options.globalNdv - supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = 0 except (ValueError, ArithmeticError) as npe: config_options.errMsg = "Unable to run mask search on MRMS supplemental precip: " + str(npe) @@ -2609,7 +2613,7 @@ def regrid_hourly_nbm(forcings_or_precip, config_options, wrf_hydro_geo_meta, mp config_options.statusMsg = "Exceeded max hours for NBM data, will not use NBM in final layering." err_handler.log_msg(config_options, mpi_config) return - + # If the expected file is missing, this means we are allowing missing files, simply # exit out of this routine as the regridded fields have already been set to NDV. if not os.path.exists(forcings_or_precip.file_in1): @@ -2707,7 +2711,7 @@ def regrid_hourly_nbm(forcings_or_precip, config_options, wrf_hydro_geo_meta, mp terrain_tmp = os.path.join(config_options.scratch_dir, 'nbm_terrain_temp.nc') cmd = f"$WGRIB2 {config_options.grid_meta} -netcdf {terrain_tmp}" - hgt_tmp = ioMod.open_grib2(config_options.grid_meta, terrain_tmp, cmd, config_options, + hgt_tmp = ioMod.open_grib2(config_options.grid_meta, terrain_tmp, cmd, config_options, mpi_config, 'DIST_surface') if mpi_config.rank == 0: var_tmp = hgt_tmp.variables['DIST_surface'][0, :, :] From 33d1f7e8e07ce1b04ed0ad9aa822585b2f3a36e3 Mon Sep 17 00:00:00 2001 From: Ryan Cabell Date: Fri, 18 Aug 2023 09:32:59 -0600 Subject: [PATCH 06/33] Set negative MRMS values to globalNdv silently * change negative value replacement from 0 to config_options.globalNdv * remove log warning (since negatives are most likely missing values in radar-only files) --- core/regrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/regrid.py b/core/regrid.py index de977a1..feeaacb 100755 --- a/core/regrid.py +++ b/core/regrid.py @@ -2045,9 +2045,9 @@ def regrid_mrms_hourly(supplemental_precip, config_options, wrf_hydro_geo_meta, # Set any pixel cells outside the input domain to the global missing value, and set negative precip values to 0 try: if len(np.argwhere(supplemental_precip.esmf_field_out.data < 0)) > 0: - supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = 0 - config_options.statusMsg = "WARNING: Found negative precipitation values in MRMS data, setting to 0" - err_handler.log_warning(config_options, mpi_config) + supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = config_options.globalNdv + # config_options.statusMsg = "WARNING: Found negative precipitation values in MRMS data, setting to missing_value" + # err_handler.log_warning(config_options, mpi_config) supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.regridded_mask == 0)] = \ config_options.globalNdv From 84eb47efb50fb191b49c563ea5c647d2f47b36e8 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Wed, 23 Aug 2023 14:41:33 -0600 Subject: [PATCH 07/33] Update tests.yml Removed line continuations ('\') in the "Set up ESMF and wgrib2" run step. They were contributing to the CI failures. --- .github/workflows/tests.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ed818d..eadf172 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,14 +34,14 @@ jobs: - name: Set up ESMF and wgrib2 run: | - mkdir $HOME/.esmf-src \ - cd $HOME/.esmf-src \ - wget http://www.earthsystemmodeling.org/esmf_releases/public/ESMF_7_1_0r/esmf_7_1_0r_src.tar.gz \ - tar xfz esmf_7_1_0r_src.tar.gz \ - cd esmf \ - export ESMF_DIR=$PWD \ - export ESMF_COMM=mpich3 \ - make \ + mkdir $HOME/.esmf-src + cd $HOME/.esmf-src + wget http://www.earthsystemmodeling.org/esmf_releases/public/ESMF_7_1_0r/esmf_7_1_0r_src.tar.gz + tar xfz esmf_7_1_0r_src.tar.gz + cd esmf + export ESMF_DIR=$PWD + export ESMF_COMM=mpich3 + make make install cd $HOME/.esmf-src/esmf/src/addon/ESMPy From e575aa780c3e927efcb578a73cc93741119cb984 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 10:46:16 -0600 Subject: [PATCH 08/33] Updated ESMF_7_1_0r source URL (now hosted on Github) --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eadf172..d1f9617 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: run: | mkdir $HOME/.esmf-src cd $HOME/.esmf-src - wget http://www.earthsystemmodeling.org/esmf_releases/public/ESMF_7_1_0r/esmf_7_1_0r_src.tar.gz + wget https://github.com/esmf-org/esmf/archive/refs/tags/ESMF_7_1_0r.tar.gz tar xfz esmf_7_1_0r_src.tar.gz cd esmf export ESMF_DIR=$PWD From 814e8566935db0bf4787561836053d5470767058 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 10:50:42 -0600 Subject: [PATCH 09/33] Wrong spelling of ESMF tarball filename. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1f9617..68a8933 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: mkdir $HOME/.esmf-src cd $HOME/.esmf-src wget https://github.com/esmf-org/esmf/archive/refs/tags/ESMF_7_1_0r.tar.gz - tar xfz esmf_7_1_0r_src.tar.gz + tar xfz ESMF_7_1_0r.tar.gz cd esmf export ESMF_DIR=$PWD export ESMF_COMM=mpich3 From f6767971ceae3e5e1f44ca20a2e414c7dc53d16f Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 11:02:43 -0600 Subject: [PATCH 10/33] More paths fixed in tests.yml --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68a8933..303c280 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,14 +38,14 @@ jobs: cd $HOME/.esmf-src wget https://github.com/esmf-org/esmf/archive/refs/tags/ESMF_7_1_0r.tar.gz tar xfz ESMF_7_1_0r.tar.gz - cd esmf + cd esmf-ESMF_7_1_0r export ESMF_DIR=$PWD export ESMF_COMM=mpich3 make make install - cd $HOME/.esmf-src/esmf/src/addon/ESMPy - python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install + cd $HOME/.esmf-src/esmf-ESMF_7_1_0r/src/addon/ESMPy + python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf-ESMF_7_1_0r/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install cd $HOME wget ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz From bb1d23a315aaa83284f9e1e5598e8ee5ce381b39 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 11:12:21 -0600 Subject: [PATCH 11/33] More tests.yml fixes in conda setup --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 303c280..7f66341 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 with: - channel: 'defaults' # Specify the Conda channel you want to use + channels: defaults # Specify the Conda channel you want to use python-version: 3.8 # Specify the Python version for the Conda environment - name: Set up Conda and initialize shell @@ -29,7 +29,7 @@ jobs: conda init bash conda create -n esmpy_env eval "$(conda shell.bash hook)" - source activate esmpy_env + conda activate esmpy_env shell: bash - name: Set up ESMF and wgrib2 From ce4c23fc3e854c2b9831e2211badaa4ee0c5c861 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 11:23:04 -0600 Subject: [PATCH 12/33] Added ubuntu install of mpich --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f66341..4a9ab9c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,8 @@ jobs: - name: Set up ESMF and wgrib2 run: | + apt-get install mpich + mkdir $HOME/.esmf-src cd $HOME/.esmf-src wget https://github.com/esmf-org/esmf/archive/refs/tags/ESMF_7_1_0r.tar.gz From cf3d0258f799c15ed97ad7ac51acac92c8c9a567 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 11:25:04 -0600 Subject: [PATCH 13/33] Argh. Need sudo for installing packages on runner machine --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4a9ab9c..3c782dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: - name: Set up ESMF and wgrib2 run: | - apt-get install mpich + sudo apt-get install mpich mkdir $HOME/.esmf-src cd $HOME/.esmf-src From 985be5e52b3c50f472b86e6b01131fe858633443 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 11:40:42 -0600 Subject: [PATCH 14/33] Trying newer version of ESMF --- .github/workflows/tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3c782dd..1407383 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,16 +38,17 @@ jobs: mkdir $HOME/.esmf-src cd $HOME/.esmf-src - wget https://github.com/esmf-org/esmf/archive/refs/tags/ESMF_7_1_0r.tar.gz - tar xfz ESMF_7_1_0r.tar.gz - cd esmf-ESMF_7_1_0r + wget https://github.com/esmf-org/esmf/archive/refs/tags/v8.5.0.tar.gz + tar xfz v8.5.0.tar.gz + ln -s esmf-8.5.0 esmf + cd esmf export ESMF_DIR=$PWD export ESMF_COMM=mpich3 make make install - cd $HOME/.esmf-src/esmf-ESMF_7_1_0r/src/addon/ESMPy - python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf-ESMF_7_1_0r/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install + cd $HOME/.esmf-src/esmf/src/addon/ESMPy + python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install cd $HOME wget ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz From e69082f9491b66a57358ef819c7e0fc27f4fa8f4 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 12:42:07 -0600 Subject: [PATCH 15/33] More mods for building ESMFpy and wgrib2 --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1407383..e700b21 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,13 +47,13 @@ jobs: make make install - cd $HOME/.esmf-src/esmf/src/addon/ESMPy - python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install + cd $HOME/.esmf-src/esmf/src/addon/esmpy + make install cd $HOME - wget ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz - tar xfz wgrib2.tgz - rm wgrib2.tgz + wget https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v3.1.2 + tar xfz wgrib2.tgz.v3.1.2 + rm wgrib2.tgz.v3.1.2 cd grib2 export CC=gcc export FC=gfortran From 075a21fee4b87a912f49ae12a952892913b86215 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Thu, 24 Aug 2023 13:14:53 -0600 Subject: [PATCH 16/33] wgrib2 and ESMF working, trying to get pytest working --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e700b21..71c1bf8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -68,7 +68,10 @@ jobs: python setup.py install - name: Run pytest - run: pytest + run: | + pwd + cd core/tests + pytest # Optional: Generate test coverage report - name: Install coverage tool From 2bad68c4e4c97e0d412974c3c71d1b2e4b280b70 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Fri, 25 Aug 2023 10:42:41 -0600 Subject: [PATCH 17/33] Added downloading of test data from google drive to tests. Removed dead config_data symlinks and config_data folder. Now if the folder doesn't exist it triggers the download to populate it. Added hook to allow either esmpy or ESMF python library name (name changed in ESMPy v 8.4.0) --- .github/workflows/tests.yml | 7 ++- core/Regridding/regrid_base.py | 7 ++- core/geoMod.py | 7 ++- core/regrid.py | 6 ++- core/tests/config_data/DOMAIN_Default | 1 - core/tests/config_data/HIRESW.Hawaii | 1 - core/tests/config_data/NAM.Hawaii | 1 - core/tests/config_data/Puerto_Rico | 1 - core/tests/gdrive_download.py | 73 +++++++++++++++++++++++++++ core/tests/test_config_options.py | 4 ++ core/tests/test_input_forcings.py | 4 ++ genForcing.py | 8 ++- 12 files changed, 107 insertions(+), 13 deletions(-) delete mode 120000 core/tests/config_data/DOMAIN_Default delete mode 120000 core/tests/config_data/HIRESW.Hawaii delete mode 120000 core/tests/config_data/NAM.Hawaii delete mode 120000 core/tests/config_data/Puerto_Rico create mode 100644 core/tests/gdrive_download.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 71c1bf8..27c3af1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: channels: defaults # Specify the Conda channel you want to use python-version: 3.8 # Specify the Python version for the Conda environment - - name: Set up Conda and initialize shell + - name: Initialize conda environment and activate in shell run: | conda init bash conda create -n esmpy_env @@ -32,7 +32,7 @@ jobs: conda activate esmpy_env shell: bash - - name: Set up ESMF and wgrib2 + - name: Install mpich, ESMF and wgrib2 run: | sudo apt-get install mpich @@ -60,7 +60,7 @@ jobs: make make lib - - name: Install dependencies + - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt # If you have a requirements.txt file @@ -69,7 +69,6 @@ jobs: - name: Run pytest run: | - pwd cd core/tests pytest diff --git a/core/Regridding/regrid_base.py b/core/Regridding/regrid_base.py index c47803a..d7e2442 100644 --- a/core/Regridding/regrid_base.py +++ b/core/Regridding/regrid_base.py @@ -3,7 +3,12 @@ from abc import ABC, abstractmethod from contextlib import contextmanager -import ESMF +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + import numpy as np from core import err_handler diff --git a/core/geoMod.py b/core/geoMod.py index eda307c..722df13 100755 --- a/core/geoMod.py +++ b/core/geoMod.py @@ -1,6 +1,11 @@ import math -import ESMF +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + import numpy as np from netCDF4 import Dataset diff --git a/core/regrid.py b/core/regrid.py index 2534613..8cb1342 100755 --- a/core/regrid.py +++ b/core/regrid.py @@ -2,7 +2,11 @@ """ Regridding module file for regridding input forcing files. """ -import ESMF +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF import os import sys import traceback diff --git a/core/tests/config_data/DOMAIN_Default b/core/tests/config_data/DOMAIN_Default deleted file mode 120000 index 54150f2..0000000 --- a/core/tests/config_data/DOMAIN_Default +++ /dev/null @@ -1 +0,0 @@ -/glade/p/cisl/nwc/nwmv30_finals/Hawaii/DOMAIN_Default \ No newline at end of file diff --git a/core/tests/config_data/HIRESW.Hawaii b/core/tests/config_data/HIRESW.Hawaii deleted file mode 120000 index 14010bb..0000000 --- a/core/tests/config_data/HIRESW.Hawaii +++ /dev/null @@ -1 +0,0 @@ -/glade/scratch/zhangyx/ForcingEngine/Forcing_Inputs/HIRESW.Hawaii \ No newline at end of file diff --git a/core/tests/config_data/NAM.Hawaii b/core/tests/config_data/NAM.Hawaii deleted file mode 120000 index e6a9e17..0000000 --- a/core/tests/config_data/NAM.Hawaii +++ /dev/null @@ -1 +0,0 @@ -/glade/scratch/zhangyx/ForcingEngine/Forcing_Inputs/NAM.Hawaii \ No newline at end of file diff --git a/core/tests/config_data/Puerto_Rico b/core/tests/config_data/Puerto_Rico deleted file mode 120000 index e6744c6..0000000 --- a/core/tests/config_data/Puerto_Rico +++ /dev/null @@ -1 +0,0 @@ -/glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Puerto_Rico \ No newline at end of file diff --git a/core/tests/gdrive_download.py b/core/tests/gdrive_download.py new file mode 100644 index 0000000..d858391 --- /dev/null +++ b/core/tests/gdrive_download.py @@ -0,0 +1,73 @@ +from argparse import ArgumentParser + +import requests +import os + + +def maybe_download_testdata(): + # See if we've downloaded the test data from + # google drive yet, and download it if not + if os.path.isdir("config_data"): + return + + id = "1lh45R0PZKqoKR5AuJF2USDrS0ipVbSMt" + outfile = "config_data.tar.gz" + + download_file_from_google_drive(id, outfile) + os.system("tar xzf %s" % outfile) + os.remove(outfile) + + +def download_file_from_google_drive(id, destination): + print('downloading google drive file id ' + id + ' to ' + destination) + URL = "https://docs.google.com/uc?export=download" + + session = requests.Session() + + response = session.get(URL, params={'id': id, 'alt': 'media', 'confirm':'t'} + , stream=True) + token = get_confirm_token(response) + + if token: + params = {'id': id, 'confirm': token} + response = session.get(URL, params=params, stream=True) + + save_response_content(response, destination) + + + +def get_confirm_token(response): + for key, value in response.cookies.items(): + if key.startswith('download_warning'): + return value + + return None + + +def save_response_content(response, destination): + CHUNK_SIZE = 32768 + + with open(destination, "wb") as f: + for chunk in response.iter_content(CHUNK_SIZE): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + +def main(): + + parser = ArgumentParser() + parser.add_argument("--file_id", + dest="file_id", + help="Google drive file ID. Get from shareable link") + parser.add_argument("--dest_file", + dest="dest_file", + help="Full path including filename for downloaded file.") + + args = parser.parse_args() + file_id = args.file_id + dest_file = args.dest_file + + download_file_from_google_drive(file_id, dest_file) + +if __name__ == "__main__": + main() diff --git a/core/tests/test_config_options.py b/core/tests/test_config_options.py index 7ec8aab..4f8f86a 100644 --- a/core/tests/test_config_options.py +++ b/core/tests/test_config_options.py @@ -1,5 +1,6 @@ import pytest from core.config import ConfigOptions +from gdrive_download import maybe_download_testdata @pytest.mark.parametrize('invalid_config', [ # Invalid values or types @@ -7,6 +8,7 @@ # Add more invalid configurations ]) def test_read_config_invalid(invalid_config): + maybe_download_testdata() with pytest.raises(Exception): config_options = ConfigOptions(invalid_config) config_options.read_config() @@ -18,6 +20,7 @@ def test_read_config_invalid(invalid_config): # Add more missing keys ]) def test_read_config_missing(missing_key): + maybe_download_testdata() config = 'yaml/configOptions_missing.yaml' with pytest.raises(Exception): config_options = ConfigOptions(config) @@ -29,6 +32,7 @@ def test_read_config_missing(missing_key): # Add more empty values ]) def test_read_config_empty(empty_value): + maybe_download_testdata() with pytest.raises(Exception): config_options = ConfigOptions(empty_value) config_options.read_config() diff --git a/core/tests/test_input_forcings.py b/core/tests/test_input_forcings.py index 2837218..a5de23d 100644 --- a/core/tests/test_input_forcings.py +++ b/core/tests/test_input_forcings.py @@ -1,14 +1,18 @@ import pytest from core.config import ConfigOptions from core.forcingInputMod import input_forcings +from gdrive_download import maybe_download_testdata + @pytest.fixture def config_options(): + maybe_download_testdata() config_path = './yaml/configOptions_valid.yaml' yield ConfigOptions(config_path) @pytest.fixture def input_forcings_sample(config_options): + maybe_download_testdata() config_options.read_config() force_key = config_options.input_forcings[0] InputDict = {} diff --git a/genForcing.py b/genForcing.py index 1ac2fd9..cf5d4fd 100755 --- a/genForcing.py +++ b/genForcing.py @@ -1,8 +1,12 @@ import argparse import os -import ESMF - +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + from core import config from core import err_handler from core import forcingInputMod From 134b2df6c58546ba2b361dca6eeea35dd01f1094 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Fri, 25 Aug 2023 11:16:06 -0600 Subject: [PATCH 18/33] Changed np.int to int --- core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config.py b/core/config.py index e8307ab..3c822ef 100755 --- a/core/config.py +++ b/core/config.py @@ -430,7 +430,7 @@ def read_config(self): # Read in the temperature downscaling options. # Create temporary array to hold flags of if we need input parameter files. - param_flag = np.empty([len(self.input_forcings)], np.int) + param_flag = np.empty([len(self.input_forcings)], int) param_flag[:] = 0 try: self.t2dDownscaleOpt = [input['Downscaling']['Temperature'] for input in inputs] From 90230cd639363d9411c7ddc4a5646b36c502ac5e Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 10:50:25 -0600 Subject: [PATCH 19/33] Added run_tests.py to download test data first before running pytest. Updated CI test to run tests out of a docker container instead of building everything as part of the workflow (building ESMF was taking > 20 minutes). Container name is currently andygaydos/wrf_hydro_forcing - this will need to be changed to point to final location of docker image. --- .github/workflows/tests.yml | 83 ++++++++----------------------- core/tests/gdrive_download.py | 2 +- core/tests/run_tests.py | 13 +++++ core/tests/test_config_options.py | 4 -- core/tests/test_input_forcings.py | 3 -- setup.py | 2 +- 6 files changed, 36 insertions(+), 71 deletions(-) create mode 100644 core/tests/run_tests.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 27c3af1..cd72763 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,73 +11,32 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Specify the desired Python version - - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + uses: actions/checkout@v3 with: - channels: defaults # Specify the Conda channel you want to use - python-version: 3.8 # Specify the Python version for the Conda environment - - - name: Initialize conda environment and activate in shell - run: | - conda init bash - conda create -n esmpy_env - eval "$(conda shell.bash hook)" - conda activate esmpy_env - shell: bash - - - name: Install mpich, ESMF and wgrib2 + path: WrfHydroForcing + + - name: Run testing container run: | - sudo apt-get install mpich - - mkdir $HOME/.esmf-src - cd $HOME/.esmf-src - wget https://github.com/esmf-org/esmf/archive/refs/tags/v8.5.0.tar.gz - tar xfz v8.5.0.tar.gz - ln -s esmf-8.5.0 esmf - cd esmf - export ESMF_DIR=$PWD - export ESMF_COMM=mpich3 - make - make install + docker run -t --name test_container \ + -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \ + andygaydos/wrf_hydro_forcing:latest - cd $HOME/.esmf-src/esmf/src/addon/esmpy - make install + # - name: Copy test results from container + # if: ${{ always() }} + # run: docker cp test_container:/home/docker/test_out $GITHUB_WORKSPACE/test_report - cd $HOME - wget https://www.ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz.v3.1.2 - tar xfz wgrib2.tgz.v3.1.2 - rm wgrib2.tgz.v3.1.2 - cd grib2 - export CC=gcc - export FC=gfortran - make - make lib - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt # If you have a requirements.txt file - conda install -c conda-forge esmpy - python setup.py install - - - name: Run pytest - run: | - cd core/tests - pytest + # - name: Run pytest + # run: | + # cd core/tests + # pytest - # Optional: Generate test coverage report - - name: Install coverage tool - run: pip install coverage + # # Optional: Generate test coverage report + # - name: Install coverage tool + # run: pip install coverage - - name: Run pytest with coverage - run: coverage run -m pytest + # - name: Run pytest with coverage + # run: coverage run -m pytest - - name: Generate coverage report - run: coverage html -i + # - name: Generate coverage report + # run: coverage html -i diff --git a/core/tests/gdrive_download.py b/core/tests/gdrive_download.py index d858391..7b28bde 100644 --- a/core/tests/gdrive_download.py +++ b/core/tests/gdrive_download.py @@ -10,7 +10,7 @@ def maybe_download_testdata(): if os.path.isdir("config_data"): return - id = "1lh45R0PZKqoKR5AuJF2USDrS0ipVbSMt" + id = "1qZbSpgUbi6Is5VlJzsvZ3R5w1WtA1uS3" outfile = "config_data.tar.gz" download_file_from_google_drive(id, outfile) diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py new file mode 100644 index 0000000..7d5cde2 --- /dev/null +++ b/core/tests/run_tests.py @@ -0,0 +1,13 @@ +from gdrive_download import maybe_download_testdata +import subprocess + + +def run(): + maybe_download_testdata() + + cmd = "pytest" + subprocess.run(cmd, shell=True) + + +if __name__ == "__main__": + run() \ No newline at end of file diff --git a/core/tests/test_config_options.py b/core/tests/test_config_options.py index 4f8f86a..7ec8aab 100644 --- a/core/tests/test_config_options.py +++ b/core/tests/test_config_options.py @@ -1,6 +1,5 @@ import pytest from core.config import ConfigOptions -from gdrive_download import maybe_download_testdata @pytest.mark.parametrize('invalid_config', [ # Invalid values or types @@ -8,7 +7,6 @@ # Add more invalid configurations ]) def test_read_config_invalid(invalid_config): - maybe_download_testdata() with pytest.raises(Exception): config_options = ConfigOptions(invalid_config) config_options.read_config() @@ -20,7 +18,6 @@ def test_read_config_invalid(invalid_config): # Add more missing keys ]) def test_read_config_missing(missing_key): - maybe_download_testdata() config = 'yaml/configOptions_missing.yaml' with pytest.raises(Exception): config_options = ConfigOptions(config) @@ -32,7 +29,6 @@ def test_read_config_missing(missing_key): # Add more empty values ]) def test_read_config_empty(empty_value): - maybe_download_testdata() with pytest.raises(Exception): config_options = ConfigOptions(empty_value) config_options.read_config() diff --git a/core/tests/test_input_forcings.py b/core/tests/test_input_forcings.py index a5de23d..8eb0aa2 100644 --- a/core/tests/test_input_forcings.py +++ b/core/tests/test_input_forcings.py @@ -1,18 +1,15 @@ import pytest from core.config import ConfigOptions from core.forcingInputMod import input_forcings -from gdrive_download import maybe_download_testdata @pytest.fixture def config_options(): - maybe_download_testdata() config_path = './yaml/configOptions_valid.yaml' yield ConfigOptions(config_path) @pytest.fixture def input_forcings_sample(config_options): - maybe_download_testdata() config_options.read_config() force_key = config_options.input_forcings[0] InputDict = {} diff --git a/setup.py b/setup.py index d7fd30d..fb1fef2 100755 --- a/setup.py +++ b/setup.py @@ -9,5 +9,5 @@ author='Ishita Srivastava', author_email='ishitas@ucar.edu', description='', - install_requires=['netCDF4', 'numpy', 'mpi4py','ESMPy'] + install_requires=['netCDF4', 'numpy', 'mpi4py','esmpy'] ) From 38896303251ccf9a41746c36dd9d6f3a2e6a9e1c Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 11:21:09 -0600 Subject: [PATCH 20/33] Changed URL for gdrive download. Added run step to change permissions on local repo clone. Added propagation of exit status to python scripts --- .github/workflows/tests.yml | 1 + core/tests/gdrive_download.py | 13 ++----------- core/tests/run_tests.py | 5 +++-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd72763..657fd60 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,6 +17,7 @@ jobs: - name: Run testing container run: | + chmod -r 0777 /home/docker/WrfHydroForcing && \ docker run -t --name test_container \ -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \ andygaydos/wrf_hydro_forcing:latest diff --git a/core/tests/gdrive_download.py b/core/tests/gdrive_download.py index 7b28bde..f683578 100644 --- a/core/tests/gdrive_download.py +++ b/core/tests/gdrive_download.py @@ -20,22 +20,13 @@ def maybe_download_testdata(): def download_file_from_google_drive(id, destination): print('downloading google drive file id ' + id + ' to ' + destination) - URL = "https://docs.google.com/uc?export=download" + URL = f"https://drive.google.com/file/d/{id}/view?usp=drive_link" session = requests.Session() - - response = session.get(URL, params={'id': id, 'alt': 'media', 'confirm':'t'} - , stream=True) - token = get_confirm_token(response) - - if token: - params = {'id': id, 'confirm': token} - response = session.get(URL, params=params, stream=True) - + response = session.get(URL, stream=True) save_response_content(response, destination) - def get_confirm_token(response): for key, value in response.cookies.items(): if key.startswith('download_warning'): diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py index 7d5cde2..b09b154 100644 --- a/core/tests/run_tests.py +++ b/core/tests/run_tests.py @@ -6,8 +6,9 @@ def run(): maybe_download_testdata() cmd = "pytest" - subprocess.run(cmd, shell=True) + test_proc = subprocess.run(cmd, shell=True) + exit(test_proc.returncode) if __name__ == "__main__": - run() \ No newline at end of file + run() From 4d4c13903787782b3670176d47a66b83e638332c Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 11:23:45 -0600 Subject: [PATCH 21/33] Bug fix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 657fd60..e7c2344 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: - name: Run testing container run: | - chmod -r 0777 /home/docker/WrfHydroForcing && \ + chmod -R 0777 $GITHUB_WORKSPACE/WrfHydroForcing && \ docker run -t --name test_container \ -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \ andygaydos/wrf_hydro_forcing:latest From 5765d7460cffe5e921b7c5abe8412bda49f189bf Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 11:48:17 -0600 Subject: [PATCH 22/33] Whoops gdrive mod didn't work... --- core/tests/gdrive_download.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/tests/gdrive_download.py b/core/tests/gdrive_download.py index f683578..5a4646a 100644 --- a/core/tests/gdrive_download.py +++ b/core/tests/gdrive_download.py @@ -20,10 +20,18 @@ def maybe_download_testdata(): def download_file_from_google_drive(id, destination): print('downloading google drive file id ' + id + ' to ' + destination) - URL = f"https://drive.google.com/file/d/{id}/view?usp=drive_link" + URL = "https://docs.google.com/uc?export=download" session = requests.Session() - response = session.get(URL, stream=True) + + response = session.get(URL, params={'id': id, 'alt': 'media', 'confirm':'t'} + , stream=True) + token = get_confirm_token(response) + + if token: + params = {'id': id, 'confirm': token} + response = session.get(URL, params=params, stream=True) + save_response_content(response, destination) From d382e14a66903eaa4e5362dc145e9f4018d9f66b Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 29 Aug 2023 13:56:15 -0600 Subject: [PATCH 23/33] Changed docker image tag name to wrfhydro/wrf_hydro_forcing:latest --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e7c2344..2fcaa1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: chmod -R 0777 $GITHUB_WORKSPACE/WrfHydroForcing && \ docker run -t --name test_container \ -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \ - andygaydos/wrf_hydro_forcing:latest + wrfhydro/wrf_hydro_forcing:latest # - name: Copy test results from container # if: ${{ always() }} From 405659e02b737be30499c9649056aa791e0efe1f Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Fri, 1 Sep 2023 09:29:54 -0600 Subject: [PATCH 24/33] Changed np.int and np.bool to int and bool respectively, as the former produce errors. Additional YAML key checks (using try/except blocks). Updated ioMod.py to remove temporary grib2.tbl file from scratch dir when finished with it. Uncommented NETCDF variable definition, as it was causing 'not defined' errors --- core/config.py | 26 +++++++++++++++++++++++--- core/disaggregateMod.py | 6 +++--- core/downscale.py | 8 +++++--- core/ioMod.py | 5 +++++ core/parallel.py | 4 ++-- core/time_handling.py | 4 ++-- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/core/config.py b/core/config.py index 3c822ef..d3676b1 100755 --- a/core/config.py +++ b/core/config.py @@ -140,7 +140,18 @@ def read_config(self): self.number_inputs = len(inputs) # Create Enums dynamically from the yaml files - self.forcingInputModYaml = config['YamlConfig']['forcingInputModYaml'] + try: + yaml_config = config['YamlConfig'] + except KeyError: + raise KeyError("Unable to locate YamlConfig in configuration file.") + err_handler.err_out_screen('Unable to locate YamlConfig in configuration file.') + + try: + self.forcingInputModYaml = yaml_config['forcingInputModYaml'] + except KeyError: + raise KeyError("Unable to locate forcingInputModYaml in configuration file.") + err_handler.err_out_screen('Unable to locate forcingInputModYaml in configuration file.') + forcing_yaml_stream = open(self.forcingInputModYaml) forcingConfig = yaml.safe_load(forcing_yaml_stream) dynamicForcing = {} @@ -148,15 +159,24 @@ def read_config(self): dynamicForcing[k] = k self.ForcingEnum = enum.Enum('ForcingEnum', dynamicForcing) - self.suppPrecipModYaml = config['YamlConfig']['suppPrecipModYaml'] + try: + self.suppPrecipModYaml = yaml_config['suppPrecipModYaml'] + except KeyError: + raise KeyError("Unable to locate suppPrecipModYaml in configuration file.") + err_handler.err_out_screen('Unable to locate suppPrecipModYaml in configuration file.') + supp_yaml_stream = open(self.suppPrecipModYaml) suppConfig = yaml.safe_load(supp_yaml_stream) dynamicSupp = {} for k in suppConfig.keys(): dynamicSupp[k] = k self.SuppForcingPcpEnum = enum.Enum('SuppForcingPcpEnum', dynamicSupp) + try: + self.outputVarAttrYaml = yaml_config['outputVarAttrYaml'] + except KeyError: + raise KeyError("Unable to locate outputVarAttrYaml in configuration file.") + err_handler.err_out_screen('Unable to locate outputVarAttrYaml in configuration file.') - self.outputVarAttrYaml = config['YamlConfig']['outputVarAttrYaml'] out_yaml_stream = open(self.outputVarAttrYaml) outConfig = yaml.safe_load(out_yaml_stream) dynamicOut = {} diff --git a/core/disaggregateMod.py b/core/disaggregateMod.py index f86d731..ad4281d 100755 --- a/core/disaggregateMod.py +++ b/core/disaggregateMod.py @@ -114,9 +114,9 @@ def ext_ana_disaggregate(input_forcings, supplemental_precip, config_options, mp ana_sum = np.array([],dtype=np.float32) target_data = np.array([],dtype=np.float32) - ana_all_zeros = np.array([],dtype=np.bool) - ana_no_zeros = np.array([],dtype=np.bool) - target_data_no_zeros = np.array([],dtype=np.bool) + ana_all_zeros = np.array([],dtype=bool) + ana_no_zeros = np.array([],dtype=bool) + target_data_no_zeros = np.array([],dtype=bool) if mpi_config.rank == 0: config_options.statusMsg = f"Performing hourly disaggregation of {supplemental_precip.file_in2}" err_handler.log_msg(config_options, mpi_config) diff --git a/core/downscale.py b/core/downscale.py index 8de6986..ad6abb4 100755 --- a/core/downscale.py +++ b/core/downscale.py @@ -764,15 +764,17 @@ def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,ConfigOptions,COSZEN,declin COSZEN[np.where(COSZEN < 1E-4)] = 1E-4 - corr_frac = np.empty([ny, nx], np.int) - # shadow_mask = np.empty([ny,nx],np.int) - diffuse_frac = np.empty([ny, nx], np.int) + corr_frac = np.empty([ny, nx], int) + + # shadow_mask = np.empty([ny,nx],int) + diffuse_frac = np.empty([ny, nx], int) corr_frac[:, :] = 0 diffuse_frac[:, :] = 0 # shadow_mask[:,:] = 0 indTmp = np.where((GeoMetaWrfHydro.slope[:,:] == 0.0) & (SWDOWN <= 10.0)) + corr_frac[indTmp] = 1 term1 = np.sin(xxlat) * np.cos(hrang2d) diff --git a/core/ioMod.py b/core/ioMod.py index a5c0ae7..05ef921 100755 --- a/core/ioMod.py +++ b/core/ioMod.py @@ -535,6 +535,11 @@ def open_grib2(GribFileIn,NetCdfFileOut,Wgrib2Cmd,ConfigOptions,MpiConfig, err = None exitcode = None + # remove temporary grib2.tbl file + g2path = os.path.join(ConfigOptions.scratch_dir, "grib2.tbl") + if os.path.isfile(g2path): + os.remove(g2path) + # Ensure file exists. if not os.path.isfile(NetCdfFileOut): ConfigOptions.errMsg = "Expected NetCDF file: " + NetCdfFileOut + \ diff --git a/core/parallel.py b/core/parallel.py index 910b578..f93ef8d 100755 --- a/core/parallel.py +++ b/core/parallel.py @@ -140,7 +140,7 @@ def scatter_array_scatterv_no_cache(self,geoMeta,src_array,ConfigOptions): data_type_flag = 1 if src_array.dtype == np.float64: data_type_flag = 2 - if src_array.dtype == np.bool: + if src_array.dtype == bool: data_type_flag = 3 # Broadcast the data_type_flag to other processors @@ -206,7 +206,7 @@ def scatter_array_scatterv_no_cache(self,geoMeta,src_array,ConfigOptions): recvbuf=np.empty([counts[self.rank]],np.float32) elif data_type_flag == 3: data_type = MPI.BOOL - recvbuf = np.empty([counts[self.rank]], np.bool) + recvbuf = np.empty([counts[self.rank]], bool) else: data_type = MPI.DOUBLE recvbuf = np.empty([counts[self.rank]], np.float64) diff --git a/core/time_handling.py b/core/time_handling.py index 5c94122..52f314f 100755 --- a/core/time_handling.py +++ b/core/time_handling.py @@ -8,8 +8,8 @@ import numpy as np from strenum import StrEnum from core import err_handler -#from core.forcingInputMod import input_forcings -#NETCDF = input_forcings.NETCDF +from core.forcingInputMod import input_forcings +NETCDF = input_forcings.NETCDF def calculate_lookback_window(config_options): From cf47ae39c0231e3e41d8f9c2765429075960d0d5 Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Fri, 8 Sep 2023 11:19:24 -0600 Subject: [PATCH 25/33] Added initial mpi tests. Modified run_tests.py to use mpirun with 4 processes. Added pytest-mpi to requirements. --- core/tests/run_tests.py | 2 +- core/tests/test_mpi_meta.py | 64 +++++++++++++++++++++++++++++++++++++ requirements.txt | 4 ++- 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 core/tests/test_mpi_meta.py diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py index b09b154..b90129a 100644 --- a/core/tests/run_tests.py +++ b/core/tests/run_tests.py @@ -5,7 +5,7 @@ def run(): maybe_download_testdata() - cmd = "pytest" + cmd = "mpirun -n 4 python -m pytest --with-mpi" test_proc = subprocess.run(cmd, shell=True) exit(test_proc.returncode) diff --git a/core/tests/test_mpi_meta.py b/core/tests/test_mpi_meta.py new file mode 100644 index 0000000..9468bd2 --- /dev/null +++ b/core/tests/test_mpi_meta.py @@ -0,0 +1,64 @@ +import pytest +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro +import numpy as np + +@pytest.fixture +def config_options(): + config_path = 'yaml/configOptions_valid.yaml' + return ConfigOptions(config_path) + +@pytest.fixture +def geo_meta(config_options, mpi_meta): + config_options.read_config() + + WrfHydroGeoMeta = GeoMetaWrfHydro() + WrfHydroGeoMeta.initialize_destination_geo(config_options, mpi_meta) + WrfHydroGeoMeta.initialize_geospatial_metadata(config_options, mpi_meta) + + return WrfHydroGeoMeta + +@pytest.fixture +def mpi_meta(config_options): + config_options.read_config() + + mpi_meta = MpiConfig() + mpi_meta.initialize_comm(config_options) + + return mpi_meta + +@pytest.mark.mpi +def test_create_mpi_config(config_options, mpi_meta): + config_options.read_config() + +@pytest.mark.mpi +def test_broadcast_parameter(config_options, mpi_meta): + + # int + value = 999 if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == 999) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=int) + assert param == 999 + + # float + value = 999.999 if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == 999.999) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=float) + assert param == 999.999 + + # bool + value = True if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == True) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=bool) + assert param == True + + +@pytest.mark.mpi +def test_scatter_array(config_options, mpi_meta, geo_meta): + test_array = np.zeros([1,390,590], dtype=np.float64) + + #arr = mpi_meta.scatter_array(geo_meta,test_array,config_options) + + + assert True \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 29fdbb1..ec5986a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,11 +7,13 @@ charset-normalizer cycler decorator entrypoints +h5py ipykernel ipython -netCDF4 +netcdf4 numpy pytest +pytest-mpi pyaml PyYAML requests From 66071e0564c2dbff60c4da0c47ebbcd177cba723 Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Fri, 8 Sep 2023 14:47:31 -0600 Subject: [PATCH 26/33] Added tests and assertions for scattering an array using ESMF and mpi --- core/tests/test_mpi_meta.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/core/tests/test_mpi_meta.py b/core/tests/test_mpi_meta.py index 9468bd2..29d3ad7 100644 --- a/core/tests/test_mpi_meta.py +++ b/core/tests/test_mpi_meta.py @@ -56,9 +56,27 @@ def test_broadcast_parameter(config_options, mpi_meta): @pytest.mark.mpi def test_scatter_array(config_options, mpi_meta, geo_meta): - test_array = np.zeros([1,390,590], dtype=np.float64) - #arr = mpi_meta.scatter_array(geo_meta,test_array,config_options) + target_data = None + if mpi_meta.rank == 0: + target_data = np.random.randn(geo_meta.ny_global,geo_meta.nx_global) + # get the sum of the full grid. + value = np.sum(target_data) if mpi_meta.rank == 0 else None + value = mpi_meta.broadcast_parameter(value, config_options, param_type=float) + + target_data = mpi_meta.scatter_array(geo_meta,target_data,config_options) - assert True \ No newline at end of file + # None of the sub grids' sum should equal the sum of the full grid + # unless there's just a single process + assert np.sum(target_data) != value or mpi_meta.size == 1 + + # The x dimension size should be the same as the full grid + assert target_data.shape[1] == geo_meta.nx_global + + # The length of the y dimension of the sub grid should equal (full grid.y / num_processes), + # account for rounding when scattering the array + assert abs((target_data.shape[0] * mpi_meta.size) - geo_meta.ny_global) < mpi_meta.size + + + \ No newline at end of file From 63e8c710a5ba1d88174c92533668b9dc96f1be1b Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Mon, 11 Sep 2023 13:28:16 -0600 Subject: [PATCH 27/33] Added geoMod tests. Added mpi scatter_array tests. Modified run_tests.py to test various MPI n_procs (none, 1, 4, and 8) for consistency --- core/tests/run_tests.py | 54 ++++++++++++++++++++++++++++++++++++- core/tests/test_geo_meta.py | 37 +++++++++++++++++++++++++ core/tests/test_mpi_meta.py | 22 +-------------- 3 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 core/tests/test_geo_meta.py diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py index b90129a..d7fab47 100644 --- a/core/tests/run_tests.py +++ b/core/tests/run_tests.py @@ -1,13 +1,65 @@ from gdrive_download import maybe_download_testdata import subprocess +COLOR_GREEN = "\033[92m" +COLOR_NORM = "\033[0m" +COLOR_RED = "\033[91m" + def run(): maybe_download_testdata() + failed = 0 + + print ("Testing without MPI...") + + cmd = "python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s----Tests not using MPI failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- Tests not using MPI passed%s" % (COLOR_GREEN, COLOR_NORM)) + + print ("Testing with 1 MPI process...") + + cmd = "mpirun -n 1 python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 1 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 1 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + print ("Testing with 4 MPI processes...") + cmd = "mpirun -n 4 python -m pytest --with-mpi" test_proc = subprocess.run(cmd, shell=True) - exit(test_proc.returncode) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 4 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 4 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + + print ("Testing with 8 MPI processes...") + + cmd = "mpirun -n 8 python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 8 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 8 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + if failed > 0: + print("%s---- Total %s MPI configurations failed!!!%s" % (failed, COLOR_RED, COLOR_NORM)) + exit(-1) + + print("%s******** All tests passed ***********%s" % (COLOR_GREEN, COLOR_NORM)) + exit(0) + + if __name__ == "__main__": diff --git a/core/tests/test_geo_meta.py b/core/tests/test_geo_meta.py new file mode 100644 index 0000000..b4adecf --- /dev/null +++ b/core/tests/test_geo_meta.py @@ -0,0 +1,37 @@ +import pytest +from netCDF4 import Dataset + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro + +from fixtures import * + +import numpy as np + + +@pytest.mark.mpi +def test_get_processor_bounds(geo_meta,mpi_meta): + geo_meta.get_processor_bounds() + + assert geo_meta.x_lower_bound == 0 + assert geo_meta.x_upper_bound == geo_meta.nx_global + + assert abs((geo_meta.ny_global / mpi_meta.size) - geo_meta.ny_local) < mpi_meta.size + assert abs(geo_meta.y_lower_bound - (mpi_meta.rank * geo_meta.ny_local)) < mpi_meta.size + assert abs(geo_meta.y_upper_bound - ((mpi_meta.rank + 1) * geo_meta.ny_local)) < mpi_meta.size + +@pytest.mark.mpi +def test_calc_slope(geo_meta,config_options): + config_options.read_config() + + idTmp = Dataset(config_options.geogrid,'r') + (slopeOut,slp_azi) = geo_meta.calc_slope(idTmp,config_options) + + assert slopeOut.shape == slp_azi.shape + assert slopeOut.size == slp_azi.size + + # test the sums of these grids are as expected, + # within floating point rounding tolerance + assert abs(np.sum(slopeOut) - 1527.7666) < 0.001 + assert abs(np.sum(slp_azi) - 58115.258) < 0.001 \ No newline at end of file diff --git a/core/tests/test_mpi_meta.py b/core/tests/test_mpi_meta.py index 29d3ad7..58123dc 100644 --- a/core/tests/test_mpi_meta.py +++ b/core/tests/test_mpi_meta.py @@ -4,29 +4,9 @@ from core.geoMod import GeoMetaWrfHydro import numpy as np -@pytest.fixture -def config_options(): - config_path = 'yaml/configOptions_valid.yaml' - return ConfigOptions(config_path) +from fixtures import * -@pytest.fixture -def geo_meta(config_options, mpi_meta): - config_options.read_config() - - WrfHydroGeoMeta = GeoMetaWrfHydro() - WrfHydroGeoMeta.initialize_destination_geo(config_options, mpi_meta) - WrfHydroGeoMeta.initialize_geospatial_metadata(config_options, mpi_meta) - - return WrfHydroGeoMeta - -@pytest.fixture -def mpi_meta(config_options): - config_options.read_config() - - mpi_meta = MpiConfig() - mpi_meta.initialize_comm(config_options) - return mpi_meta @pytest.mark.mpi def test_create_mpi_config(config_options, mpi_meta): From 090cfd25fa96b5f46867dd38f221050014751c77 Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Mon, 11 Sep 2023 13:32:24 -0600 Subject: [PATCH 28/33] Forgot to add fixtures.py - a common set of pytest test fixtures --- core/tests/fixtures.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 core/tests/fixtures.py diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py new file mode 100644 index 0000000..3aaa23a --- /dev/null +++ b/core/tests/fixtures.py @@ -0,0 +1,29 @@ +import pytest + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro + +@pytest.fixture +def mpi_meta(config_options): + config_options.read_config() + + mpi_meta = MpiConfig() + mpi_meta.initialize_comm(config_options) + + return mpi_meta + +@pytest.fixture +def geo_meta(config_options, mpi_meta): + config_options.read_config() + + WrfHydroGeoMeta = GeoMetaWrfHydro() + WrfHydroGeoMeta.initialize_destination_geo(config_options, mpi_meta) + WrfHydroGeoMeta.initialize_geospatial_metadata(config_options, mpi_meta) + + return WrfHydroGeoMeta + +@pytest.fixture +def config_options(): + config_path = './yaml/configOptions_valid.yaml' + yield ConfigOptions(config_path) \ No newline at end of file From c242506f55c7902f7a3c620b4a910a94e5383e99 Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Wed, 13 Sep 2023 14:36:05 -0600 Subject: [PATCH 29/33] Added bias correction tests --- core/tests/fixtures.py | 26 ++++- core/tests/run_tests.py | 6 +- core/tests/test_bias_correction.py | 147 +++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 core/tests/test_bias_correction.py diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py index 3aaa23a..0cc540b 100644 --- a/core/tests/fixtures.py +++ b/core/tests/fixtures.py @@ -3,6 +3,17 @@ from core.parallel import MpiConfig from core.config import ConfigOptions from core.geoMod import GeoMetaWrfHydro +from core.forcingInputMod import input_forcings, initDict + +import numpy as np + + +def isWithinTolerance(val1, val2, tolerance=None): + if tolerance is None: + tolerance = 0.0 + + diff = abs(val1 - val2) + return diff <= tolerance @pytest.fixture def mpi_meta(config_options): @@ -26,4 +37,17 @@ def geo_meta(config_options, mpi_meta): @pytest.fixture def config_options(): config_path = './yaml/configOptions_valid.yaml' - yield ConfigOptions(config_path) \ No newline at end of file + yield ConfigOptions(config_path) + +@pytest.fixture +def input_forcings_final(config_options,geo_meta): + config_options.read_config() + + inputForcingMod = initDict(config_options,geo_meta) + input_forcings = inputForcingMod[config_options.input_forcings[0]] + + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 + + return input_forcings + diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py index d7fab47..df8362f 100644 --- a/core/tests/run_tests.py +++ b/core/tests/run_tests.py @@ -11,15 +11,15 @@ def run(): failed = 0 - print ("Testing without MPI...") + print ("Testing without mpirun...") cmd = "python -m pytest --with-mpi" test_proc = subprocess.run(cmd, shell=True) if test_proc.returncode != 0: - print("%s----Tests not using MPI failed!!!%s" % (COLOR_RED, COLOR_NORM)) + print("%s----Tests not using mpirun failed!!!%s" % (COLOR_RED, COLOR_NORM)) failed += 1 else: - print("%s---- Tests not using MPI passed%s" % (COLOR_GREEN, COLOR_NORM)) + print("%s---- Tests not using mpirun passed%s" % (COLOR_GREEN, COLOR_NORM)) print ("Testing with 1 MPI process...") diff --git a/core/tests/test_bias_correction.py b/core/tests/test_bias_correction.py new file mode 100644 index 0000000..b5dbb81 --- /dev/null +++ b/core/tests/test_bias_correction.py @@ -0,0 +1,147 @@ +import pytest +import numpy as np + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro +from core.forcingInputMod import * + +from fixtures import * +import core.bias_correction as BC + +@pytest.mark.mpi +def test_no_bias_correct(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + final_forcings = np.copy(input_forcings_final.final_forcings) + + BC.no_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) == np.sum(input_forcings_final.final_forcings) + assert input_forcings_final.final_forcings[4][0][0] == 1.5 + +@pytest.mark.mpi +def test_ncar_tbl_correction(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_tbl_correction(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval,2.5, None) + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_blanket_adjustment_lw(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 6.768526039220487, None ) + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct(config_options, geo_meta, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_sw_hrrr_bias_correct(input_forcings_final, geo_meta, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 1.4999999948202003, None ) + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_temp_hrrr_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 1.558319002854755, None) + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_temp_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 0.7468065367962597, None) + + config_options.current_output_step = 180 + input_forcings_final.final_forcings = np.copy(final_forcings) + BC.ncar_temp_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 0.9868065367962597, None) + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_lwdown_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 12.54182912750415, None) + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct(config_options, mpi_meta, input_forcings_final): + config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + final_forcings = np.copy(input_forcings_final.final_forcings) + BC.ncar_wspd_hrrr_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + + assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + + pointval = input_forcings_final.final_forcings[4][0][0] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, 1.5722859839330992, None) + +# @pytest.mark.mpi +# def test_ncar_wspd_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): +# config_options.read_config() +# config_options.current_output_date = config_options.b_date_proc +# config_options.current_output_step = 60 + +# input_forcings_final.define_product() +# final_forcings = np.copy(input_forcings_final.final_forcings) +# BC.ncar_wspd_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + +# assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) + +# pointval = input_forcings_final.final_forcings[4][0][0] +# # TODO set acceptable tolerance +# assert isWithinTolerance(pointval, 1.5722859839330992, None) \ No newline at end of file From 5be220e0b4a267a8d86e016c8cf43da0774369f8 Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Mon, 18 Sep 2023 15:23:45 -0600 Subject: [PATCH 30/33] Fixed bug in bias_correction.py (variable referenced before assignment, and indexing an array with a string instead of a number). Reworked test fixtures to more easily test outputs of various configurations (time of day, day of year, AnA vs forecast) --- core/bias_correction.py | 10 +- core/tests/fixtures.py | 21 +- core/tests/run_tests.py | 3 +- core/tests/test_bias_correction.py | 677 ++++++++++++++++++++++++----- 4 files changed, 586 insertions(+), 125 deletions(-) diff --git a/core/bias_correction.py b/core/bias_correction.py index ae04932..3f9853c 100755 --- a/core/bias_correction.py +++ b/core/bias_correction.py @@ -44,6 +44,7 @@ def run_bias_correction(input_forcings, config_options, geo_meta_wrf_hydro, mpi_ str(BiasCorrTempEnum.GFS.name) : ncar_temp_gfs_bias_correct, str(BiasCorrTempEnum.HRRR.name) : ncar_temp_hrrr_bias_correct } + bias_correct_temperature[input_forcings.t2dBiasCorrectOpt](input_forcings, config_options, mpi_config, 0) err_handler.check_program_status(config_options, mpi_config) @@ -794,6 +795,7 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force ugrd_idx = input_forcings.grib_vars.index('UGRD') vgrd_idx = input_forcings.grib_vars.index('VGRD') + OutputEnum = config_options.OutputEnum for ind, xvar in enumerate(OutputEnum): if xvar.name == input_forcings.input_map_output[ugrd_idx]: u_in = ind @@ -820,20 +822,19 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force # TODO: cache the "other" value so we don't repeat this calculation unnecessarily bias_corrected = ugrid_out if force_num == ugrd_idx else vgrid_out - OutputEnum = config_options.OutputEnum for ind, xvar in enumerate(OutputEnum): if xvar.name == input_forcings.input_map_output[force_num]: outId = ind break wind_in = None try: - wind_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + wind_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming windspeed from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) - + ind_valid = None try: ind_valid = np.where(wind_in != config_options.globalNdv) @@ -852,7 +853,7 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = wind_in[:, :] + input_forcings.final_forcings[outId, :, :] = wind_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary windspeed array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -1079,6 +1080,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for nldas_param_1 = None nldas_param_2 = None nldas_zero_pcp = None + err_handler.check_program_status(config_options, mpi_config) # Scatter NLDAS parameters diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py index 0cc540b..1989966 100644 --- a/core/tests/fixtures.py +++ b/core/tests/fixtures.py @@ -3,18 +3,10 @@ from core.parallel import MpiConfig from core.config import ConfigOptions from core.geoMod import GeoMetaWrfHydro -from core.forcingInputMod import input_forcings, initDict import numpy as np -def isWithinTolerance(val1, val2, tolerance=None): - if tolerance is None: - tolerance = 0.0 - - diff = abs(val1 - val2) - return diff <= tolerance - @pytest.fixture def mpi_meta(config_options): config_options.read_config() @@ -37,17 +29,8 @@ def geo_meta(config_options, mpi_meta): @pytest.fixture def config_options(): config_path = './yaml/configOptions_valid.yaml' - yield ConfigOptions(config_path) - -@pytest.fixture -def input_forcings_final(config_options,geo_meta): + config_options = ConfigOptions(config_path) config_options.read_config() + yield config_options - inputForcingMod = initDict(config_options,geo_meta) - input_forcings = inputForcingMod[config_options.input_forcings[0]] - - input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) - input_forcings.fcst_hour2 = 12 - - return input_forcings diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py index df8362f..8be98ed 100644 --- a/core/tests/run_tests.py +++ b/core/tests/run_tests.py @@ -41,8 +41,7 @@ def run(): else: print("%s---- MPI nprocs = 4 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) - - print ("Testing with 8 MPI processes...") + # print ("Testing with 8 MPI processes...") cmd = "mpirun -n 8 python -m pytest --with-mpi" test_proc = subprocess.run(cmd, shell=True) diff --git a/core/tests/test_bias_correction.py b/core/tests/test_bias_correction.py index b5dbb81..8bd75d6 100644 --- a/core/tests/test_bias_correction.py +++ b/core/tests/test_bias_correction.py @@ -1,147 +1,624 @@ import pytest import numpy as np +from datetime import datetime from core.parallel import MpiConfig from core.config import ConfigOptions from core.geoMod import GeoMetaWrfHydro from core.forcingInputMod import * +from core.enumConfig import * from fixtures import * import core.bias_correction as BC -@pytest.mark.mpi -def test_no_bias_correct(config_options, mpi_meta, input_forcings_final): + +def date(datestr): + return datetime.strptime(datestr, '%Y-%m-%d %H:%M:%S') + +expected_values = { + 'no_bias_correct': [ + { 'index': (0,0), 'value': 1.5, 'tolerance': None} + ], + 'ncar_tbl_correction': [ + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 1.85, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 1.85, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 1.6, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 1.6, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 1.52, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 1.52, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 1.45, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 1.45, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 18} + ], + 'ncar_blanket_adjustment_lw': [ + { 'index': (0,0), 'value': 9.82194214668789, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 7.763851170382089, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 6.8580578533121095, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 8.91614882961791, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 7.479010060854788, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 4.6895996292499, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 6.160989939145213, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 8.950400370750101, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_sw_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.5826259963214397, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4999999948202003, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4999999948202003, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.540613193064928, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4394609276205301, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.500000003795177, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.500000003795177, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.4702432015910745, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_temp_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.462288117950048, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.49941014460614, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.5757118820499518, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.53858985539386, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 0.9518785484037021, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.8684798631054194, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.884121451596298, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.9675201368945807, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_temp_gfs_bias_correct': [ + { 'index': (0,0), 'value': 2.6484931133084233, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 2.1467845464398003, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 0.23150688669157682, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 0.7332154535601993, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'ncar_lwdown_gfs_bias_correct': [ + { 'index': (0,0), 'value': 10.897517774766143, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 12.813333511002988, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 11.902482225233857, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 9.986666488997013, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'ncar_wspd_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.7324061943600948, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.6027854243986395, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.592862924985717, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.7224836949471725, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.2131465997822362, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.080473211999704, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.1504572971498712, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.2831306849324031, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_wspd_gfs_bias_correct': [ + { 'index': (0,0), 'value': 1.560235849440387, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 1.2559415578811088, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 1.1569214380849937, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 1.4612157296442718, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ] +} + +####################################################### +# +# This test suite creates a mock 'forcing grid' filled +# with a known value, calls each bias_correction function, +# and samples a gridpoint to compare against the expected +# bias-corrected value for that function. +# +# The functions currently expect an exact (full precision) +# match, but a tolerance can be set for each test or +# globally in the run_and_compare() function +# +####################################################### + +@pytest.fixture +def unbiased_input_forcings(config_options,geo_meta): + """ + Prepare the mock input forcing grid + """ config_options.read_config() - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.no_bias_correct(input_forcings_final, config_options, mpi_meta, 0) + inputForcingMod = initDict(config_options,geo_meta) + input_forcings = inputForcingMod[config_options.input_forcings[0]] + + # put a value of 1.5 at every gridpoint, for 10 mock variables + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 - assert np.sum(final_forcings) == np.sum(input_forcings_final.final_forcings) - assert input_forcings_final.final_forcings[4][0][0] == 1.5 + return input_forcings +# input_map_output: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'SWDOWN', 'LWDOWN', 'PSFC'] +# OutputEnum: {'U2D': 'U2D', 'V2D': 'V2D', 'LWDOWN': 'LWDOWN', 'RAINRATE': 'RAINRATE', 'T2D': 'T2D', 'Q2D': 'Q2D', 'PSFC': 'PSFC', 'SWDOWN': 'SWDOWN', 'LQFRAC': 'LQFRAC'} + +##### No bias correction tests ######### @pytest.mark.mpi -def test_ncar_tbl_correction(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc +def test_no_bias_correct(config_options, mpi_meta, unbiased_input_forcings): + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='no_bias_correct', equal=True) - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_tbl_correction(input_forcings_final, config_options, mpi_meta, 0) - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) +##### NCAR table bias correction tests ######### +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval,2.5, None) +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') @pytest.mark.mpi -def test_ncar_blanket_adjustment_lw(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc +def test_ncar_tbl_correction_0z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_blanket_adjustment_lw(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 6.768526039220487, None ) - @pytest.mark.mpi -def test_ncar_sw_hrrr_bias_correct(config_options, geo_meta, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc +def test_ncar_tbl_correction_6z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_sw_hrrr_bias_correct(input_forcings_final, geo_meta, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 1.4999999948202003, None ) +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') @pytest.mark.mpi -def test_ncar_temp_hrrr_bias_correct(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc +def test_ncar_tbl_correction_6z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_temp_hrrr_bias_correct(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 1.558319002854755, None) +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') @pytest.mark.mpi -def test_ncar_temp_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc - config_options.current_output_step = 60 +def test_ncar_tbl_correction_12z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_temp_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 0.7468065367962597, None) +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') - config_options.current_output_step = 180 - input_forcings_final.final_forcings = np.copy(final_forcings) - BC.ncar_temp_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 0.9868065367962597, None) +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') +##### NCAR blanket LW Bias Correction tests ######### @pytest.mark.mpi -def test_ncar_lwdown_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc - config_options.current_output_step = 60 +def test_ncar_blanket_adjustment_lw_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_lwdown_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 12.54182912750415, None) +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') @pytest.mark.mpi -def test_ncar_wspd_hrrr_bias_correct(config_options, mpi_meta, input_forcings_final): - config_options.read_config() - config_options.current_output_date = config_options.b_date_proc - config_options.current_output_step = 60 +def test_ncar_blanket_adjustment_lw_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +##### NCAR HRRR SW bias correction tests ######### +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_0z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_6z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_12z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_18z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_0z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_6z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_12z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_18z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +##### NCAR HRRR temperature bias correction tests ######### +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') - final_forcings = np.copy(input_forcings_final.final_forcings) - BC.ncar_wspd_hrrr_bias_correct(input_forcings_final, config_options, mpi_meta, 0) +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') - assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - - pointval = input_forcings_final.final_forcings[4][0][0] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, 1.5722859839330992, None) +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +##### NCAR GFS temperature bias correction tests ######### +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +##### NCAR GFS LW bias correction tests ######### +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +##### NCAR HRRR wind speed bias correction tests ######### +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +##### NCAR GFS Wind Speed bias correction tests ######### +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_0z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_6z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_12z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_18z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +# TODO: needs CFS-specific bias grid # @pytest.mark.mpi -# def test_ncar_wspd_gfs_bias_correct(config_options, mpi_meta, input_forcings_final): -# config_options.read_config() -# config_options.current_output_date = config_options.b_date_proc -# config_options.current_output_step = 60 - -# input_forcings_final.define_product() -# final_forcings = np.copy(input_forcings_final.final_forcings) -# BC.ncar_wspd_gfs_bias_correct(input_forcings_final, config_options, mpi_meta, 0) - -# assert np.sum(final_forcings) != np.sum(input_forcings_final.final_forcings) - -# pointval = input_forcings_final.final_forcings[4][0][0] -# # TODO set acceptable tolerance -# assert isWithinTolerance(pointval, 1.5722859839330992, None) \ No newline at end of file +# def test_cfsv2_nldas_nwm_bias_correct(config_options, unbiased_input_forcings, mpi_meta): +# run_and_compare(BC.cfsv2_nldas_nwm_bias_correct, config_options, unbiased_input_forcings, mpi_meta, compval=1.1326198378488306) + + +def test_run_bias_correction1(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.t2dBiasCorrectOpt = BiasCorrTempEnum.HRRR.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_temp_hrrr_bias_correct", varname='T2D') + +def test_run_bias_correction2(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.windBiasCorrectOpt = BiasCorrWindEnum.HRRR.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_wspd_hrrr_bias_correct", varname='U2D') + +def test_run_bias_correction3(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.q2dBiasCorrectOpt = BiasCorrHumidEnum.CUSTOM.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_tbl_correction", varname='Q2D') + +############ Helper functions ################ + +def run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=None, + type='ncar_sw_hrrr_bias_correct', var='SWDOWN', equal=False): + for spec in expected_values[type]: + if 'fcst_hour' in spec and spec['fcst_hour'] != unbiased_input_forcings.fcst_hour2: + continue + if 'var' in spec and spec['var'] != var: + continue + if 'ana' in spec and spec['ana'] != config_options.ana_flag: + continue + if 'date' in spec and spec['date'] != config_options.b_date_proc: + continue + + funct = getattr(BC, type) + + run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=geo_meta, + compval=spec['value'], tolerance=spec['tolerance'], index=spec['index'], + var=var, equal=equal) + +def run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, specname, varname="T2D"): + config_options.current_output_step = 60 + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(unbiased_input_forcings.final_forcings) + + BC.run_bias_correction(unbiased_input_forcings,config_options,geo_meta,mpi_meta) + assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) + for spec in expected_values[specname]: + if 'fcst_hour' in spec and spec['fcst_hour'] != unbiased_input_forcings.fcst_hour2: + continue + if 'var' in spec and spec['var'] != varname: + continue + if 'ana' in spec and spec['ana'] != config_options.ana_flag: + continue + if 'date' in spec and spec['date'] != config_options.b_date_proc: + continue + + outId = getOutputId(config_options,unbiased_input_forcings, varname) + index = (outId, spec['index'][0], spec['index'][1]) + pointval = unbiased_input_forcings.final_forcings[index] + assert isWithinTolerance(pointval, spec['value'], spec['tolerance']) + +def run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=None, + compval=None, tolerance=None, equal=False, index=(0,0), var='T2D'): + """ + Run the provided bias_correction function, and sample the output to compare + against the expected value. Also ensure the overall original and bias_corrected grid sums are different. + :param funct: The function to call + :param config_options: + :param unbiased_input_forcings: + :param mpi_meta: + :param geo_meta: Can be None if not needed for function call + :param compval: Expected corrected value. Can be None to skip this test + :tolerance: Ensure that abs(compval-sampledval) <= tolerance. If None, the values must match exactly + :param equal: If true, the bias_corrected grid should be equivalent to the original grid + """ + #config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + unbiased_input_forcings.define_product() + final_forcings = np.copy(unbiased_input_forcings.final_forcings) + varnum = unbiased_input_forcings.input_map_output.index(var) + if geo_meta: + funct(unbiased_input_forcings, geo_meta, config_options, mpi_meta, varnum) + else: + funct(unbiased_input_forcings, config_options, mpi_meta, varnum) + + if equal: + assert np.sum(final_forcings) == np.sum(unbiased_input_forcings.final_forcings) + else: + assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) + + if compval is not None: + outId = getOutputId(config_options, unbiased_input_forcings, var) + idx = (outId, index[0], index[1]) + + pointval = unbiased_input_forcings.final_forcings[idx] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, compval, tolerance) + +def getOutputId(config_options, input_forcings, varname): + OutputEnum = config_options.OutputEnum + varnum = input_forcings.input_map_output.index(varname) + outId = 0 + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[varnum]: + outId = ind + break + + return outId + +def isWithinTolerance(val1, val2, tolerance=None): + """ + Make sure val1==val2, within the provided tolerance + """ + if tolerance is None: + tolerance = 0.0 + + diff = abs(val1 - val2) + return diff <= tolerance From 90f65b1a27d4731719ad5b48741bb5a12f93c8ee Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Thu, 28 Sep 2023 13:00:22 -0600 Subject: [PATCH 31/33] Added bias correction tests for CFS. Fixed yaml configuration bugs related to CFS-specific parameters in config.py and bias_correction.py --- core/bias_correction.py | 15 +++-- core/config.py | 16 +++--- core/tests/fixtures.py | 7 +++ core/tests/test_bias_correction.py | 77 +++++++++++++++++++++++--- core/tests/yaml/configOptions_cfs.yaml | 69 +++++++++++++++++++++++ 5 files changed, 162 insertions(+), 22 deletions(-) create mode 100644 core/tests/yaml/configOptions_cfs.yaml diff --git a/core/bias_correction.py b/core/bias_correction.py index 3f9853c..7bf98f8 100755 --- a/core/bias_correction.py +++ b/core/bias_correction.py @@ -1317,12 +1317,15 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for config_options.statusMsg = "Looping over local arrays to calculate bias corrections." err_handler.log_msg(config_options, mpi_config) # Process each of the pixel cells for this local processor on the CFS grid. + outId = None + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break for x_local in range(0, input_forcings.nx_local): for y_local in range(0, input_forcings.ny_local): - cfs_prev_tmp = input_forcings.coarse_input_forcings1[input_forcings.input_map_output[force_num], - y_local, x_local] - cfs_next_tmp = input_forcings.coarse_input_forcings2[input_forcings.input_map_output[force_num], - y_local, x_local] + cfs_prev_tmp = input_forcings.coarse_input_forcings1[outId, y_local, x_local] + cfs_next_tmp = input_forcings.coarse_input_forcings2[outId, y_local, x_local] # Check for any missing parameter values. If any missing values exist, # set this flag to False. Further down, if it's False, we will simply @@ -1544,9 +1547,9 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = \ + input_forcings.final_forcings[outId, :, :] = \ input_forcings.esmf_field_out.data except NumpyExceptions as npe: config_options.errMsg = "Unable to extract ESMF field data for CFSv2: " + str(npe) err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) \ No newline at end of file diff --git a/core/config.py b/core/config.py index d3676b1..ac022a5 100755 --- a/core/config.py +++ b/core/config.py @@ -421,25 +421,25 @@ def read_config(self): # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on # for ALL variables. if self.runCfsNldasBiasCorrect: - if set(self.precipBiasCorrectOpt) != {'CFS_V2'}: + if self.precipBiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'Precipitation under this configuration.') - if min(self.lwBiasCorrectOpt) != {'CFS_V2'}: + if self.lwBiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'long-wave radiation under this configuration.') - if min(self.swBiasCorrectOpt) != {'CFS_V2'}: + if self.swBiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'short-wave radiation under this configuration.') - if min(self.t2BiasCorrectOpt) != {'CFS_V2'}: + if self.t2BiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'surface temperature under this configuration.') - if min(self.windBiasCorrect) != {'CFS_V2'}: + if self.windBiasCorrect != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'wind forcings under this configuration.') - if min(self.q2BiasCorrectOpt) != {'CFS_V2'}: + if self.q2BiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'specific humidity under this configuration.') - if min(self.psfcBiasCorrectOpt) != {'CFS_V2'}: + if self.psfcBiasCorrectOpt != ['CFS_V2']: err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' 'surface pressure under this configuration.') # Make sure we don't have any other forcings activated. This can only be ran for CFSv2. @@ -1078,7 +1078,7 @@ def read_config(self): for optTmp in self.input_forcings: if optTmp == 'CFS_V2': try: - self.cfsv2EnsMember = input['Ensembles']['cfsEnsNumber'] + self.cfsv2EnsMember = [input['Ensembles']['cfsEnsNumber'] for input in inputs if 'cfsEnsNumber' in input['Ensembles']][0] except KeyError: err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles ' 'section of the configuration file') diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py index 1989966..f87e21a 100644 --- a/core/tests/fixtures.py +++ b/core/tests/fixtures.py @@ -33,4 +33,11 @@ def config_options(): config_options.read_config() yield config_options +@pytest.fixture +def config_options_cfs(): + config_path = './yaml/configOptions_cfs.yaml' + config_options = ConfigOptions(config_path) + config_options.read_config() + yield config_options + diff --git a/core/tests/test_bias_correction.py b/core/tests/test_bias_correction.py index 8bd75d6..232adba 100644 --- a/core/tests/test_bias_correction.py +++ b/core/tests/test_bias_correction.py @@ -15,6 +15,17 @@ def date(datestr): return datetime.strptime(datestr, '%Y-%m-%d %H:%M:%S') +var_list = { + 'T2D': 300, + 'Q2D': 50.0, + 'U2D': 2.5, + 'V2D': 1.5, + 'RAINRATE': 2.5, + 'SWDOWN': 400, + 'LWDOWN': 400, + 'PSFC': 90000 +} + expected_values = { 'no_bias_correct': [ { 'index': (0,0), 'value': 1.5, 'tolerance': None} @@ -94,6 +105,16 @@ def date(datestr): { 'index': (0,0), 'value': 1.2559415578811088, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, { 'index': (0,0), 'value': 1.1569214380849937, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, { 'index': (0,0), 'value': 1.4612157296442718, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'cfsv2_nldas_nwm_bias_correct': [ + { 'index': (0,0), 'value': 285, 'tolerance': None, 'var': 'T2D' }, + { 'index': (0,0), 'value': 0.009524456432886543, 'tolerance': None, 'var': 'Q2D' }, + { 'index': (0,0), 'value': 3.562956632266148, 'tolerance': None, 'var': 'U2D' }, + { 'index': (0,0), 'value': 0.11510974533140078, 'tolerance': None, 'var': 'V2D' }, + { 'index': (0,0), 'value': 0.01622283237712793, 'tolerance': None, 'var': 'RAINRATE' }, + { 'index': (0,0), 'value': 0, 'tolerance': None, 'var': 'SWDOWN' }, + { 'index': (0,0), 'value': 272, 'tolerance': None, 'var': 'LWDOWN' }, + { 'index': (0,0), 'value': 49999 , 'tolerance': None, 'var': 'PSFC' } ] } @@ -126,8 +147,34 @@ def unbiased_input_forcings(config_options,geo_meta): return input_forcings -# input_map_output: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'SWDOWN', 'LWDOWN', 'PSFC'] -# OutputEnum: {'U2D': 'U2D', 'V2D': 'V2D', 'LWDOWN': 'LWDOWN', 'RAINRATE': 'RAINRATE', 'T2D': 'T2D', 'Q2D': 'Q2D', 'PSFC': 'PSFC', 'SWDOWN': 'SWDOWN', 'LQFRAC': 'LQFRAC'} +@pytest.fixture +def unbiased_input_forcings_cfs(config_options_cfs,geo_meta,mpi_meta): + """ + Prepare the mock input forcing grid for CFS + """ + # Don't run for mpi processes > 0, since the + # CFS and NLDAS parameters are spatially dependent and we're not sure + # how we are splitting up the grid. + if mpi_meta.size > 1: + return + config_options_cfs.read_config() + + inputForcingMod = initDict(config_options_cfs,geo_meta) + input_forcings = inputForcingMod[config_options_cfs.input_forcings[0]] + # TODO Can we construct this path from the config file? Parts of the path are currently hard-coded + # im time_handling.py, however + input_forcings.file_in2 = "config_data/CFSv2/cfs.20190923/06/6hrly_grib_01/flxf2019100106.01.2019092306.grb2" + input_forcings.fcst_hour2 = 192 + input_forcings.fcst_date1 = datetime.strptime("2019100106", "%Y%m%d%H%M") + input_forcings.fcst_date2 = input_forcings.fcst_date1 + + input_forcings.regrid_inputs(config_options_cfs,geo_meta,mpi_meta) + + # put a value of 1.5 at every gridpoint, for 10 mock variables + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 + + return input_forcings ##### No bias correction tests ######### @pytest.mark.mpi @@ -499,27 +546,41 @@ def test_ncar_wspd_gfs_bias_correct_18z(config_options, unbiased_input_forcings, config_options.b_date_proc = date('2023-10-01 18:00:00') run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') -# TODO: needs CFS-specific bias grid -# @pytest.mark.mpi -# def test_cfsv2_nldas_nwm_bias_correct(config_options, unbiased_input_forcings, mpi_meta): -# run_and_compare(BC.cfsv2_nldas_nwm_bias_correct, config_options, unbiased_input_forcings, mpi_meta, compval=1.1326198378488306) - - +##### CFS bias correction tests ######### +@pytest.mark.mpi +def test_cfsv2_nldas_nwm_bias_correct(config_options_cfs, unbiased_input_forcings_cfs, mpi_meta,geo_meta): + # Don't run for mpi processes > 0, since the + # CFS and NLDAS parameters are spatially dependent and we're not sure + # how we are splitting up the grid. + if mpi_meta.size > 1: + return + for var in var_list: + val = var_list[var] + unbiased_input_forcings_cfs.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + unbiased_input_forcings_cfs.coarse_input_forcings1 = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + unbiased_input_forcings_cfs.coarse_input_forcings2 = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + run_unit_test(config_options_cfs, mpi_meta, unbiased_input_forcings_cfs, type='cfsv2_nldas_nwm_bias_correct', var=var) + +##### Tests for the top-level 'run_bias_correction()' function ####### +@pytest.mark.mpi def test_run_bias_correction1(unbiased_input_forcings, config_options, geo_meta, mpi_meta): unbiased_input_forcings.t2dBiasCorrectOpt = BiasCorrTempEnum.HRRR.name run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_temp_hrrr_bias_correct", varname='T2D') +@pytest.mark.mpi def test_run_bias_correction2(unbiased_input_forcings, config_options, geo_meta, mpi_meta): unbiased_input_forcings.windBiasCorrectOpt = BiasCorrWindEnum.HRRR.name run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_wspd_hrrr_bias_correct", varname='U2D') +@pytest.mark.mpi def test_run_bias_correction3(unbiased_input_forcings, config_options, geo_meta, mpi_meta): unbiased_input_forcings.q2dBiasCorrectOpt = BiasCorrHumidEnum.CUSTOM.name run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_tbl_correction", varname='Q2D') + ############ Helper functions ################ def run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=None, diff --git a/core/tests/yaml/configOptions_cfs.yaml b/core/tests/yaml/configOptions_cfs.yaml new file mode 100644 index 0000000..5b93ed9 --- /dev/null +++ b/core/tests/yaml/configOptions_cfs.yaml @@ -0,0 +1,69 @@ +Forecast: + AnAFlag: false + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '201909230600' + RefcstEDateProc: '201910010600' + Shift: 0 + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: CFS_V2 + Longwave: CFS_V2 + Precip: CFS_V2 + Pressure: CFS_V2 + Shortwave: CFS_V2 + Temperature: CFS_V2 + Wind: CFS_V2 + Dir: config_data/CFSv2 + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: config_data/CFSv2 + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: CFS_V2 + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + Ensembles: + cfsEnsNumber: 1 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' From db652822429ecfcd82d8ee9210641c0cb11acfd0 Mon Sep 17 00:00:00 2001 From: Andy Gaydos Date: Tue, 31 Oct 2023 15:03:10 -0600 Subject: [PATCH 32/33] Refactor (#100) * Regridding subclasses with abstract class * New YAML configuration changes * Revert changes to AnA metadata handling * removed July 2022 changes to AnA metadata handling and reverted to v2.1 behavior. * removed unneeded special-case code to artificially limit Alaska AnA cycle/forecast counting * AK Short-Range NBM support (#90) * Shift NBM cycles back 3 hours for AK Short-Range to match HRRR * Fix unintentional Alaska MR NBM shift * Onboarding updates * Fix parallel bug in NBM regridding * Fix incorrect time divisor in AK ExtAnA * Add spatial metadata globals to output files (#94) * Support NBM as primary forcing input (#93) * Additional NBM support * Add support for NBM input with external terrain file * Add creep-fill for NBM * Remove precipitation from CONUS HRRR forcing (#92) * Resolved merge conflicts with the main branch * Create README.md * Change `model_total_valid_times` attrib to int32 * Change globalNdv to -999999 * Add to ignored global metadata * Don't mask NLDAS params when reading netCDF var The _FillValue in the NLDAS params needs to be explicity changed to the globalNdv value so the masked areas can be reconstructed on the other side of an MPI broadcast * Adding pytest unit tests and related changes * Adding change to python version in the workflow * Adding change to requirements file by removing ESMpy in the workflow * Adding change to requirements file by removing unnecessary packages in the workflow * Adding change to requirements file by removing unnecessary packages in the workflow * Adding change to requirements file by removing unnecessary packages in the workflow * Adding change to requirements file by removing unnecessary packages in the workflow * Adding change to python version * Adding change to linux system type * Adding change to linux system type * Adding change to linux system type * Adding change to linux system type * Adding change to linux system type * Adding change to linux system type * Removed disutils from requirements * Removed packages from requirements * Removed packages from requirements * Removed packages from requirements * Removed packages from requirements * Removed packages from requirements * Removed packages from requirements * Adding python path dependencies * Adding python path dependencies * Adding python path dependencies * Adding python path dependencies * Adding WrfHydroForcing before core in import statements * Removing WrfHydroForcing before core in import statements * change to python path * change to python path * change to setup * change to setup * change to setup * python 3.8 * python 3.8 * python 3.8 * python 3.8 * python 3.8 * Restructured the tests and added relative paths * Added conda init to github workflow * Modified conda init to github workflow * Modified conda init to github workflow * Modified conda activate to source in github workflow * Modified conda activation in a non interactive terminal in github workflow * Added Installed /opt/hostedtoolcache/Python/3.8.17/x64/lib/python3.8/site-packages/wrf_hydro_mfe-1.0-py3.8.egg in github workflow * Added ESMPy install * Added ESMPy install * Added ESMPy install * Added ESMPy install * Added ESMPy install * Added ESMPy install * Delete configOptions_empty.yaml * Delete configOptions_invalid1.yaml * Delete configOptions_invalid2.yaml * Delete configOptions_missing.yaml * Delete configOptions_missingKey.yaml * Delete configOptions_valid.yaml * Delete inputForcings_empty.yaml * Delete inputForcings_invalid.yaml * Delete inputForcings_missing.yaml * Delete inputForcings_valid.yaml * Delete report.html * Fix issues with MRMS overlaying HRRR/RAP * Re-include removed HRRR precipitaion for use in Short-Range forecasts * Fix issue with MRMS missing data when used as an overlay in AnA * Set negative MRMS values to globalNdv silently * change negative value replacement from 0 to config_options.globalNdv * remove log warning (since negatives are most likely missing values in radar-only files) * Update tests.yml Removed line continuations ('\') in the "Set up ESMF and wgrib2" run step. They were contributing to the CI failures. * Updated ESMF_7_1_0r source URL (now hosted on Github) * Wrong spelling of ESMF tarball filename. * More paths fixed in tests.yml * More tests.yml fixes in conda setup * Added ubuntu install of mpich * Argh. Need sudo for installing packages on runner machine * Trying newer version of ESMF * More mods for building ESMFpy and wgrib2 * wgrib2 and ESMF working, trying to get pytest working * Added downloading of test data from google drive to tests. Removed dead config_data symlinks and config_data folder. Now if the folder doesn't exist it triggers the download to populate it. Added hook to allow either esmpy or ESMF python library name (name changed in ESMPy v 8.4.0) * Changed np.int to int * Added run_tests.py to download test data first before running pytest. Updated CI test to run tests out of a docker container instead of building everything as part of the workflow (building ESMF was taking > 20 minutes). Container name is currently andygaydos/wrf_hydro_forcing - this will need to be changed to point to final location of docker image. * Changed URL for gdrive download. Added run step to change permissions on local repo clone. Added propagation of exit status to python scripts * Bug fix * Whoops gdrive mod didn't work... * Changed docker image tag name to wrfhydro/wrf_hydro_forcing:latest * Changed np.int and np.bool to int and bool respectively, as the former produce errors. Additional YAML key checks (using try/except blocks). Updated ioMod.py to remove temporary grib2.tbl file from scratch dir when finished with it. Uncommented NETCDF variable definition, as it was causing 'not defined' errors * Added initial mpi tests. Modified run_tests.py to use mpirun with 4 processes. Added pytest-mpi to requirements. * Added tests and assertions for scattering an array using ESMF and mpi * Added geoMod tests. Added mpi scatter_array tests. Modified run_tests.py to test various MPI n_procs (none, 1, 4, and 8) for consistency * Forgot to add fixtures.py - a common set of pytest test fixtures * Added bias correction tests * Fixed bug in bias_correction.py (variable referenced before assignment, and indexing an array with a string instead of a number). Reworked test fixtures to more easily test outputs of various configurations (time of day, day of year, AnA vs forecast) * Added bias correction tests for CFS. Fixed yaml configuration bugs related to CFS-specific parameters in config.py and bias_correction.py --------- Co-authored-by: ishita9 Co-authored-by: ishita9 <36771676+ishita9@users.noreply.github.com> Co-authored-by: ishita9 Co-authored-by: Ryan Cabell Co-authored-by: Andrew Gaydos --- .github/workflows/tests.yml | 43 + ...late_forcing_engine_Alaska_Analysis.config | 351 +++++ ...e_forcing_engine_Alaska_ExtAnalysis.config | 351 +++++ ...mplate_forcing_engine_Alaska_Medium.config | 350 +++++ ..._forcing_engine_Alaska_Medium_Blend.config | 352 +++++ ...emplate_forcing_engine_Alaska_Short.config | 346 +++++ .../template_forcing_engine_Analysis.config | 349 +++++ ...late_forcing_engine_Hawaii_Analysis.config | 341 +++++ ...emplate_forcing_engine_Hawaii_Short.config | 341 +++++ .../v3.0/template_forcing_engine_Long.config | 346 +++++ .../template_forcing_engine_Medium.config | 337 +++++ ...emplate_forcing_engine_Medium_Blend.config | 350 +++++ ..._forcing_engine_PuertoRico_Analysis.config | 333 +++++ ...ate_forcing_engine_PuertoRico_Short.config | 341 +++++ .../v3.0/template_forcing_engine_Short.config | 346 +++++ Config/YAML/forcingInputMod.yaml | 255 ++++ Config/YAML/outputVarAttr.yaml | 89 ++ Config/YAML/suppPrecipMod.yaml | 87 ++ ...mplate_forcing_engine_AK_AnA_hourly.config | 352 +++++ ...template_forcing_engine_AK_AnA_hourly.yaml | 498 +++++++ README.md | 28 + ...template_forcing_engine_AK_AnA_hourly.yaml | 63 + Util/config2yaml.py | 152 ++ Util/yaml_comment_template.py | 456 ++++++ core/Regridding/__init__.py | 0 core/Regridding/regrid_base.py | 314 +++++ core/Regridding/regrid_cfs.py | 46 + core/Regridding/regrid_grib2.py | 185 +++ core/Regridding/regrid_hrrr.py | 6 + core/Regridding/regrid_netcdf.py | 103 ++ core/Regridding/regrid_rap.py | 6 + core/bias_correction.py | 186 ++- core/config.py | 1156 +++++++-------- core/config_v1.py | 1250 +++++++++++++++++ core/disaggregateMod.py | 14 +- core/downscale.py | 117 +- core/enumConfig.py | 215 +++ core/err_handler.py | 23 +- core/forcingInputMod.py | 552 +++----- core/forecastMod.py | 15 +- core/geoMod.py | 7 +- core/ioMod.py | 93 +- core/layeringMod.py | 29 +- core/parallel.py | 4 +- core/regrid.py | 428 ++++-- core/suppPrecipMod.py | 203 +-- core/tests/assets/style.css | 177 +++ core/tests/fixtures.py | 43 + core/tests/gdrive_download.py | 72 + core/tests/run_tests.py | 65 + core/tests/test_bias_correction.py | 685 +++++++++ core/tests/test_config_options.py | 48 + core/tests/test_geo_meta.py | 37 + core/tests/test_input_forcings.py | 62 + core/tests/test_mpi_meta.py | 62 + core/tests/yaml/configOptions_cfs.yaml | 69 + core/tests/yaml/configOptions_empty.yaml | 67 + core/tests/yaml/configOptions_invalid1.yaml | 67 + core/tests/yaml/configOptions_invalid2.yaml | 67 + core/tests/yaml/configOptions_missing.yaml | 52 + core/tests/yaml/configOptions_missingKey.yaml | 59 + core/tests/yaml/configOptions_valid.yaml | 67 + core/tests/yaml/forcingInputMod.yaml | 255 ++++ core/tests/yaml/inputForcings_empty.yaml | 255 ++++ core/tests/yaml/inputForcings_invalid.yaml | 255 ++++ core/tests/yaml/inputForcings_missing.yaml | 227 +++ core/tests/yaml/inputForcings_valid.yaml | 255 ++++ core/tests/yaml/outputVarAttr.yaml | 89 ++ core/tests/yaml/suppPrecipMod.yaml | 87 ++ core/time_handling.py | 106 +- genForcing.py | 13 +- requirements.txt | 20 + setup.py | 6 +- 73 files changed, 13507 insertions(+), 1569 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Alaska_Analysis.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Alaska_ExtAnalysis.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium_Blend.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Alaska_Short.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Analysis.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Analysis.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Short.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Long.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Medium.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Medium_Blend.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Analysis.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Short.config create mode 100755 Config/WCOSS/v3.0/template_forcing_engine_Short.config create mode 100644 Config/YAML/forcingInputMod.yaml create mode 100644 Config/YAML/outputVarAttr.yaml create mode 100644 Config/YAML/suppPrecipMod.yaml create mode 100644 Config/YAML/template_forcing_engine_AK_AnA_hourly.config create mode 100644 Config/YAML/template_forcing_engine_AK_AnA_hourly.yaml create mode 100644 README.md create mode 100644 Test/AK/ana/template_forcing_engine_AK_AnA_hourly.yaml create mode 100755 Util/config2yaml.py create mode 100644 Util/yaml_comment_template.py create mode 100644 core/Regridding/__init__.py create mode 100644 core/Regridding/regrid_base.py create mode 100644 core/Regridding/regrid_cfs.py create mode 100644 core/Regridding/regrid_grib2.py create mode 100644 core/Regridding/regrid_hrrr.py create mode 100644 core/Regridding/regrid_netcdf.py create mode 100644 core/Regridding/regrid_rap.py create mode 100755 core/config_v1.py create mode 100644 core/enumConfig.py mode change 100755 => 100644 core/forcingInputMod.py mode change 100755 => 100644 core/suppPrecipMod.py create mode 100644 core/tests/assets/style.css create mode 100644 core/tests/fixtures.py create mode 100644 core/tests/gdrive_download.py create mode 100644 core/tests/run_tests.py create mode 100644 core/tests/test_bias_correction.py create mode 100644 core/tests/test_config_options.py create mode 100644 core/tests/test_geo_meta.py create mode 100644 core/tests/test_input_forcings.py create mode 100644 core/tests/test_mpi_meta.py create mode 100644 core/tests/yaml/configOptions_cfs.yaml create mode 100644 core/tests/yaml/configOptions_empty.yaml create mode 100644 core/tests/yaml/configOptions_invalid1.yaml create mode 100644 core/tests/yaml/configOptions_invalid2.yaml create mode 100644 core/tests/yaml/configOptions_missing.yaml create mode 100644 core/tests/yaml/configOptions_missingKey.yaml create mode 100644 core/tests/yaml/configOptions_valid.yaml create mode 100644 core/tests/yaml/forcingInputMod.yaml create mode 100644 core/tests/yaml/inputForcings_empty.yaml create mode 100644 core/tests/yaml/inputForcings_invalid.yaml create mode 100644 core/tests/yaml/inputForcings_missing.yaml create mode 100644 core/tests/yaml/inputForcings_valid.yaml create mode 100644 core/tests/yaml/outputVarAttr.yaml create mode 100644 core/tests/yaml/suppPrecipMod.yaml create mode 100644 requirements.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2fcaa1f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,43 @@ +name: Run pytest + +on: + push: + branches: + - refactor # Adjust the branch name as per your repository's default branch + +jobs: + pytest: + runs-on: ubuntu-latest # Use the desired operating system for the CI environment + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + path: WrfHydroForcing + + - name: Run testing container + run: | + chmod -R 0777 $GITHUB_WORKSPACE/WrfHydroForcing && \ + docker run -t --name test_container \ + -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \ + wrfhydro/wrf_hydro_forcing:latest + + # - name: Copy test results from container + # if: ${{ always() }} + # run: docker cp test_container:/home/docker/test_out $GITHUB_WORKSPACE/test_report + + + # - name: Run pytest + # run: | + # cd core/tests + # pytest + + # # Optional: Generate test coverage report + # - name: Install coverage tool + # run: pip install coverage + + # - name: Run pytest with coverage + # run: coverage run -m pytest + + # - name: Generate coverage report + # run: coverage html -i diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Analysis.config b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Analysis.config new file mode 100755 index 0000000..ef83bff --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Analysis.config @@ -0,0 +1,351 @@ + +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [19] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/hrrr/v4.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_alaska_14_1420.15526637/analysis_assim_alaska + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_alaska_14_1420.15526637 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 1 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = 180 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071400 +RefcstEDateProc = 202210071500 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [60] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/geo_em_AK.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/GEOGRID_LDASOUT_Spatial_Metadata_AK.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [10] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +#RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [4] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [4] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [2] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [2] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +#DownscalingParamDirs = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - Amir AK future option 1 +# 9 - Amir AK future option 2 +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [10] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_alaska_14_1420.15526637 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +#SuppPcpParamDir = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Alaska_ExtAnalysis.config b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_ExtAnalysis.config new file mode 100755 index 0000000..5b3c679 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_ExtAnalysis.config @@ -0,0 +1,351 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [20] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = NETCDF + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_extend_alaska_20_1659.15536600/analysis_assim_alaska + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_extend_alaska_20_1659.15536600/AkExtAnA + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_extend_alaska_20_1659.15536600 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 1 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +#LookBack = 1680 +LookBack = 1920 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210062000 +RefcstEDateProc = 202210062100 + + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [60] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/geo_em_AK.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/GEOGRID_LDASOUT_Spatial_Metadata_AK.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [10] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +#RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [0] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [0] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [0] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [0] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - Amir AK future option 1 +# 9 - Amir AK future option 2 +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS/MRMS Precip Blend +SuppPcp = [11] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_extend_alaska_20_1659.15536600/pcpanl + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium.config b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium.config new file mode 100755 index 0000000..977e595 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium.config @@ -0,0 +1,350 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +InputForcings = [3] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/gfs/v16.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_alaska_06_1455.15528918/medium_range_alaska + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_alaska_06_1455.15528918 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202201010000 +EDateProc = 202201010600 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070600 +RefcstEDateProc = 202210071200 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [14400] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/geo_em_AK.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/GEOGRID_LDASOUT_Spatial_Metadata_AK.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +# RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +# DownscalingParamDirs = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +SuppPcp = [] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = [] + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = + +# Use SuppPcpMaxHours to limit usage of supplemental precip +# SuppPcpMaxHours = 72 + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium_Blend.config b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium_Blend.config new file mode 100755 index 0000000..3aa5158 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Medium_Blend.config @@ -0,0 +1,352 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +InputForcings = [3] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/gfs/v16.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_blend_alaska_06_1133.15516247/medium_range_blend_alaska + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_blend_alaska_06_1133.15516247 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202201010000 +EDateProc = 202201010600 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070600 +RefcstEDateProc = 202210071200 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [14400] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/geo_em_AK.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/GEOGRID_LDASOUT_Spatial_Metadata_AK.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +# RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +# DownscalingParamDirs = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +SuppPcp = [9] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +#SuppPcpDirectories = /glade/scratch/zhangyx/ForcingEngine/Forcing_Inputs/Alaska.MRMS +SuppPcpDirectories = /lfs/h1/ops/prod/com/blend/v4.0 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [1] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +#SuppPcpParamDir = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +# Use SuppPcpMaxHours to limit usage of supplemental precip +# SuppPcpMaxHours = 72 + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Short.config b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Short.config new file mode 100755 index 0000000..e045fcd --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Alaska_Short.config @@ -0,0 +1,346 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +InputForcings = [19] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/hrrr/v4.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_alaska_15_1615.15533999/short_range_alaska + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_alaska_15_1615.15533999 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202109010000 +EDateProc = 202109020000 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071500 +RefcstEDateProc = 202210071800 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 180 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [2880] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/geo_em_AK.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_alaska/GEOGRID_LDASOUT_Spatial_Metadata_AK.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +# RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +SuppPcp = [9] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/ops/prod/com/blend/v4.0 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Alaska + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Analysis.config b/Config/WCOSS/v3.0/template_forcing_engine_Analysis.config new file mode 100755 index 0000000..abab309 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Analysis.config @@ -0,0 +1,349 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [6,5] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be con +InputForcingTypes = GRIB2, GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/rap/v5.1, /lfs/h1/ops/prod/com/hrrr/v4.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1,1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_14_1430.15527265/AnAout + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_14_1430.15527265 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 1 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. + +LookBack = 180 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071400 +RefcstEDateProc = 202210071500 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [60, 60] +#ForecastInputHorizons = [180, 180] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +ForecastInputOffsets = [0, 0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/geo_em_CONUS.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0,10] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1,1] + +#RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0,0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0, 4] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0,0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0,0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0, 4] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0, 2] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0, 2] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0, 0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [2, 2] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1, 1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1, 1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0, 0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1, 1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/AnA + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [1, 5] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2, GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/ops/prod/dcom/ldmdata/obs/upperair/mrms/conus, /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_14_1430.15527265 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0, 0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1, 1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0, 0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0, 0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 2 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/AnA + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Analysis.config b/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Analysis.config new file mode 100755 index 0000000..bc88c7d --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Analysis.config @@ -0,0 +1,341 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [13] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/nam/v4.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_hawaii_14_1425.15526967/HIAnA + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_hawaii_14_1425.15526967 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202004152300 +EDateProc = 202004200000 + +[Forecast] +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = 180 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071400 +RefcstEDateProc = 202210071500 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [60] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_hawaii/geo_em_HI.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_hawaii/GEOGRID_LDASOUT_Spatial_Metadata_HI.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +AnAFlag = 1 + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - Use monthly PRISM climatology regridded to the WRF-Hydro domain to +# downscale precipitation via mountain mapper. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Hawaii + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - CONUS MRMS GRIB2 hourly radar-only QPE +# 2 - CONUS MRMS GRIB2 hourly MultiSensor QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [6] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_hawaii_14_1425.15526967 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Hawaii + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = 1 + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Short.config b/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Short.config new file mode 100755 index 0000000..56dbcd4 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Hawaii_Short.config @@ -0,0 +1,341 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [16, 8] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2, GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/nam/v4.2, /lfs/h1/ops/prod/com/hiresw/v8.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1, 1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_hawaii_12_1455.15528879/short_range_hawaii + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_hawaii_12_1455.15528879 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = syyyymmdy0000 +EDateProc = eyyyymmdy0000 + +[Forecast] +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071200 +RefcstEDateProc = 202210080000 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 720 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [2880, 2880] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0, 0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_hawaii/geo_em_HI.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_hawaii/GEOGRID_LDASOUT_Spatial_Metadata_HI.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1, 1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0, 0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +AnAFlag = 0 + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0, 0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0, 0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0, 0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0, 0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0, 0] + +# Specify a bias correction for incoming long wave radiation flux. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0, 0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0, 0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [0, 1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [0, 1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1, 0] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - Use monthly PRISM climatology regridded to the WRF-Hydro domain to +# downscale precipitation via mountain mapper. +PrecipDownscaling = [0, 0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [0, 1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Hawaii, /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Hawaii + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - CONUS MRMS GRIB2 hourly radar-only QPE +# 2 - CONUS MRMS GRIB2 hourly MultiSensor QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - Hawaii MRMS GRIB2 hourly MultiSensor QPE +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [ ] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/ops/prod/com/hiresw/v8.1 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Hawaii + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = 1 + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Long.config b/Config/WCOSS/v3.0/template_forcing_engine_Long.config new file mode 100755 index 0000000..6312316 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Long.config @@ -0,0 +1,346 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [7] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/cfs/v2.3 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 180 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_long_range_mem01_06_1410.15526131/long_range + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_long_range_mem01_06_1410.15526131 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202004140600 +EDateProc = 202004141200 + +[Forecast] +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +#LookBack = 1440 +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070600 +RefcstEDateProc = 202210071200 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [43200] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/geo_em_CONUS.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. WARNING - Will result in states from +# the nearest later forecast data point will be used if output +# timesteps are in-between two input forecast points. +# 1 - Nearest temporal neighbor. +# 2 - Weighted linear average between input points. +forcingTemporalInterpolation = [2] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correctioni +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [1] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [1] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [1] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [1] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [1] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [1] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [1] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [2] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Long_Range + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - Hawaii MRMS GRIB2 hourly MultiSensor QPE +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [] +#RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = 1 + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] + diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Medium.config b/Config/WCOSS/v3.0/template_forcing_engine_Medium.config new file mode 100755 index 0000000..3d3aa63 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Medium.config @@ -0,0 +1,337 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +InputForcings = [3] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/gfs/v16.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_00_1437.15527748/medium_range + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_00_1437.15527748 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002020000 +EDateProc = 202002030000 + +[Forecast] +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +#LookBack = 1440 +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070000 +RefcstEDateProc = 202210070600 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [14400] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/geo_em_CONUS.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. WARNING - Will result in states from +# the nearest later forecast data point will be used if output +# timesteps are in-between two input forecast points. +# 1 - Nearest temporal neighbor. +# 2 - Weighted linear average between input points. +forcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correctioni +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [3] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [3] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [3] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [2] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [1] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Medium_Range + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - Hawaii MRMS GRIB2 hourly MultiSensor QPE +SuppPcp = [] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [] + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] + diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Medium_Blend.config b/Config/WCOSS/v3.0/template_forcing_engine_Medium_Blend.config new file mode 100755 index 0000000..8b7638d --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Medium_Blend.config @@ -0,0 +1,350 @@ + +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [3] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/gfs/v16.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_blend_00_1437.15527751/medium_range_blend + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_medium_range_blend_00_1437.15527751 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070000 +RefcstEDateProc = 202210070600 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [14400] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/geo_em_CONUS.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] + +#RegridWeightsDir = /glade/scratch/zhangyx/ForcingEngine/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [3] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [3] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [3] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [2] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [1] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Medium_Range + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - Amir AK future option 1 +# 9 - Amir AK future option 2 +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [8] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/ops/prod/com/blend/v4.0 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Medium_Range + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Analysis.config b/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Analysis.config new file mode 100755 index 0000000..934db37 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Analysis.config @@ -0,0 +1,333 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +InputForcings = [14] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/nam/v4.2 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_puertorico_14_1425.15526916/PRAnA + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_puertorico_14_1425.15526916 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202004152300 +EDateProc = 202004200000 + +[Forecast] +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = 180 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071400 +RefcstEDateProc = 202210071500 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [60] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_puertorico/geo_em_PRVI.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_puertorico/GEOGRID_LDASOUT_Spatial_Metadata_PRVI.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +AnAFlag = 1 + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0] + +# Specify a bias correction for incoming long wave radiation flux. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - Use monthly PRISM climatology regridded to the WRF-Hydro domain to +# downscale precipitation via mountain mapper. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/PuertoRico + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly MultiSensor QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - Hawaii MRMS GRIB2 hourly MultiSensor QPE +SuppPcp = [5] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = GRIB2 + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_analysis_assim_puertorico_14_1425.15526916 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [6] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/PuertoRico + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = 1 + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Short.config b/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Short.config new file mode 100755 index 0000000..51330f9 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_PuertoRico_Short.config @@ -0,0 +1,341 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 PRVI nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - PRVI 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_PRVI_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [17, 18] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2, GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/nam/v4.2, /lfs/h1/ops/prod/com/hiresw/v8.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1, 1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_puertorico_06_1721.15537991/short_range_puertorico + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_puertorico_06_1721.15537991 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = syyyymmdy0000 +EDateProc = eyyyymmdy0000 + +[Forecast] +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210070600 +RefcstEDateProc = 202210071800 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 720 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [2880, 2880] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0, 6] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_puertorico/geo_em_PRVI.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain_puertorico/GEOGRID_LDASOUT_Spatial_Metadata_PRVI.nc + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1, 1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0, 0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +AnAFlag = 0 + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0, 0] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0, 0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0, 0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0, 0] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0, 0] + +# Specify a bias correction for incoming long wave radiation flux. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0, 0] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0, 0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [0, 1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [0, 1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1, 0] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - Use monthly PRISM climatology regridded to the WRF-Hydro domain to +# downscale precipitation via mountain mapper. +PrecipDownscaling = [0, 0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [0, 1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/PuertoRico, /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/PuertoRico + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - CONUS MRMS GRIB2 hourly radar-only QPE +# 2 - CONUS MRMS GRIB2 hourly MultiSensor QPE +# 3 - WRF-ARW 2.5 km 48-hr PRVI nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - PRVI MRMS GRIB2 hourly MultiSensor QPE +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [ ] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /lfs/h1/ops/prod/com/hiresw/v8.1 + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/PuertoRico + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = 1 + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/WCOSS/v3.0/template_forcing_engine_Short.config b/Config/WCOSS/v3.0/template_forcing_engine_Short.config new file mode 100755 index 0000000..1ea0451 --- /dev/null +++ b/Config/WCOSS/v3.0/template_forcing_engine_Short.config @@ -0,0 +1,346 @@ +#-------------------------------------------------------------------- +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [6, 5] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = GRIB2, GRIB2 + +# Specify the input directories for each forcing product. +InputForcingDirectories = /lfs/h1/ops/prod/com/rap/v5.1, /lfs/h1/ops/prod/com/hrrr/v4.1 + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1, 1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_13_1430.15527256/short_range + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /lfs/h1/owp/ptmp/cham.pham/test/tmp/nwm_forcing_short_range_13_1430.15527256 + +# Flag to activate scale_factor / add_offset byte packing in +# the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 0 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = -9999 +#LookBack = 1800 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202210071300 +RefcstEDateProc = 202210071900 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 360 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +ForecastInputHorizons = [1080, 1080] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +ForecastInputOffsets = [0, 0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/geo_em_CONUS.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/domain/GEOGRID_LDASOUT_Spatial_Metadata_CONUS.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [0,10] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1, 1] + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0, 0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [0, 4] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0, 0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0, 0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [0, 4] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [0, 2] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [0, 2] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0, 0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [2, 2] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1, 1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1, 1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0, 0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1, 1] + +# Specify the input parameter directory containing necessary downscaling grids. +DownscalingParamDirs = /lfs/h1/owp/nwm/noscrub/cham.pham/test/packages/nwm.v3.0.0/parm/forcingParam/Short_Range + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - Hawaii MRMS GRIB2 hourly MultiSensor QPE +# 6 - Hawaii MRMS GRIB2 hourly MultiSensor QPE (Pass 2 or Pass 1) +# 7 - MRMS SBCv2 Liquid Water Fraction (netCDF only) +# 8 - NBM Conus MR +# 9 - NBM Alaska MR +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +SuppPcpParamDir = + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/YAML/forcingInputMod.yaml b/Config/YAML/forcingInputMod.yaml new file mode 100644 index 0000000..4b7c081 --- /dev/null +++ b/Config/YAML/forcingInputMod.yaml @@ -0,0 +1,255 @@ +NLDAS: + product_name : NLDAS2_GRIB1 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF','DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +GFS_GLOBAL: + product_name : GFS_Production_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in : ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +HRRR: + product_name : HRRR_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/Config/YAML/outputVarAttr.yaml b/Config/YAML/outputVarAttr.yaml new file mode 100644 index 0000000..6a038be --- /dev/null +++ b/Config/YAML/outputVarAttr.yaml @@ -0,0 +1,89 @@ +U2D : + ind : 0 + units : 'm s-1' + standard_name : 'x_wind' + long_name : '10-m U-component of wind' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +V2D : + ind : 1 + units : 'm s-1' + standard_name : 'y_wind' + long_name : '10-m V-component of wind' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +LWDOWN : + ind : 2 + units : 'W m-2' + standard_name : 'surface_downward_longwave_flux' + long_name : 'Surface downward long-wave radiation flux' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +RAINRATE : + ind : 3 + units : 'mm s^-1' + standard_name : 'precipitation_flux' + long_name : 'Surface Precipitation Rate' + cell_methods : 'time: MEAN' + scale_factor : 1.0 + add_offset : 0.0 + least_significant_digit : 0 + +T2D : + ind : 4 + units : 'K' + standard_name : 'air_temperature' + long_name : '2-m Air Temperature' + cell_methods : 'time: point' + scale_factor : 0.01 + add_offset : 100.0 + least_significant_digit : 2 + +Q2D : + ind : 5 + units : 'kg kg-1' + standard_name : 'surface_specific_humidity' + long_name : '2-m Specific Humidity' + cell_methods : 'time: point' + scale_factor : 0.000001 + add_offset : 0.0 + least_significant_digit : 6 + +PSFC : + ind : 6 + units : 'Pa' + standard_name : 'air_pressure' + long_name : 'Surface Pressure' + cell_methods : 'time: point' + scale_factor : 0.1 + add_offset : 0.0 + least_significant_digit : 1 + +SWDOWN : + ind : 7 + units : 'W m-2' + standard_name : 'surface_downward_shortwave_flux' + long_name : 'Surface downward short-wave radiation flux' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +LQFRAC : + ind : 8 + units : '%' + standard_name : 'liquid_water_fraction' + long_name : 'Fraction of precipitation that is liquid vs. frozen' + cell_methods : 'time: point' + scale_factor : 0.1 + add_offset : 0.0 + least_significant_digit : 3 diff --git a/Config/YAML/suppPrecipMod.yaml b/Config/YAML/suppPrecipMod.yaml new file mode 100644 index 0000000..34bb47a --- /dev/null +++ b/Config/YAML/suppPrecipMod.yaml @@ -0,0 +1,87 @@ +MRMS: + product_name : MRMS_1HR_Radar_Only + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['RadarOnlyQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : ['RadarQualityIndex_0mabovemeansealevel'] + output_variables : RAINRATE + +MRMS_GAGE: + product_name : MRMS_1HR_Gage_Corrected + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['GaugeCorrQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : ['RadarQualityIndex_0mabovemeansealevel'] + output_variables : RAINRATE + +WRF_ARW_HI: + product_name : WRF_ARW_Hawaii_2p5km_PCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_2p5km_PCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_CONUS_MS: + product_name : CONUS_MRMS_1HR_MultiSensor + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_HI_MS: + product_name : Hawaii_MRMS_1HR_MultiSensor + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_SBCV2: + product_name : MRMS_LiquidWaterFraction + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['sbcv2_lwf'] + netcdf_rqi_variables : None + output_variables : LQFRAC + +AK_OPT1: + product_name : NBM_CORE_CONUS_APCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_OPT2: + product_name : NBM_CORE_ALASKA_APCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_MRMS: + product_name : AK_MRMS + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_NWS_IV: + product_name : AK_Stage_IV_Precip-MRMS + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : [] + netcdf_rqi_variables : None + output_variables : RAINRATE diff --git a/Config/YAML/template_forcing_engine_AK_AnA_hourly.config b/Config/YAML/template_forcing_engine_AK_AnA_hourly.config new file mode 100644 index 0000000..83471e0 --- /dev/null +++ b/Config/YAML/template_forcing_engine_AK_AnA_hourly.config @@ -0,0 +1,352 @@ + +# WRF-Hydro Forcing Engine Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. + +[Input] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. The following is a +# mapping of numeric values to external input native forcing files: +# 1 - NLDAS GRIB retrospective files +# 2 - NARR GRIB retrospective files +# 3 - GFS GRIB2 Global production files on the full gaussian grid +# 4 - NAM Nest GRIB2 Conus production files +# 5 - HRRR GRIB2 Conus production files +# 6 - RAP GRIB2 Conus 13km production files +# 7 - CFSv2 6-hourly GRIB2 Global production files +# 8 - WRF-ARW - GRIB2 Hawaii nest files +# 9 - GFS GRIB2 Global production files on 0.25 degree lat/lon grids. +# 10 - Custom NetCDF hourly forcing files +# 11 - Custom NetCDF hourly forcing files +# 12 - Custom NetCDF hourly forcing files +# 13 - Hawaii 3-km NAM Nest. +# 14 - Puerto Rico 3-km NAM Nest. +# 15 - Alaska 3-km Alaska NAM Nest +# 16 - NAM_Nest_3km_Hawaii_Radiation-Only +# 17 - NAM_Nest_3km_PuertoRico_Radiation-Only +# 18 - WRF-ARW GRIB2 PuertoRico +# 19 - HRRR GRIB2 Alaska production files +# 20 - ExtAna HRRR AK FE output +InputForcings = [19] + +# Specify the file type for each forcing (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +InputForcingTypes = [GRIB2] + +# Specify the input directories for each forcing product. +InputForcingDirectories = /glade/p/ral/allral/zhangyx/HRRR_Alaska + +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +InputMandatory = [1] + +[Output] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +OutputFrequency = 60 + +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +OutDir = /glade/p/ral/wsap/petzke/fe_ak/AnA/data + +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +ScratchDir = /glade/p/ral/wsap/petzke/fe_ak/AnA/data + +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# 0 - Use scale/offset encoding +# 1 - Use floating-point encoding +floatOutput = 0 + +# Flag to activate netCDF4 deflate compression in the output files. +# 0 - Deactivate compression +# 1 - Activate compression +compressOutput = 0 + +[Retrospective] +# Specify to process forcings in retrosective mode +# 0 - No +# 1 - Yes +RetroFlag = 0 + +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +BDateProc = 202002040000 +EDateProc = 202002050000 + +[Forecast] +# Specify if this is an Analysis and Assimilation run (AnA). +# If this is AnA run, set AnAFlag to 1, otherwise 0. +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +AnAFlag = 1 + +# ONLY for realtime forecasting. +# - Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +LookBack = 180 + +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +RefcstBDateProc = 202206052300 +RefcstEDateProc = 202206060000 + +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. +ForecastFrequency = 60 + +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +ForecastShift = 0 + +# Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +#ForecastInputHorizons = [60] +ForecastInputHorizons = [180] + +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +ForecastInputOffsets = [0] + +[Geospatial] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +GeogridIn = /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/geo_em.d02.nc + +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +SpatialMetaIn = /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/GEOGRID_LDASOUT_Spatial_Metadata.nc + +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +IgnoredBorderWidths = [10] + +[Regridding] +# Choose regridding options for each input forcing files being used. Options available are: +# 1 - ESMF Bilinear +# 2 - ESMF Nearest Neighbor +# 3 - ESMF Conservative Bilinear +RegridOpt = [1] +#RegridWeightsDir = /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles + +[Interpolation] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# 0 - No temporal interpolation. +# 1 - Nearest Neighbor +# 2 - Linear weighted average +ForcingTemporalInterpolation = [0] + +[BiasCorrection] +# Choose bias correction options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature bias correction method. +# 0 - No bias correction +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +TemperatureBiasCorrection = [4] + +# Specify a surface pressure bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PressureBiasCorrection = [0] + +# Specify a specific humidity bias correction method. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +HumidityBiasCorrection = [0] + +# Specify a wind bias correction. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +# 4 - NCAR parametric HRRR bias correction +WindBiasCorrection = [4] + +# Specify a bias correction for incoming short wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +SwBiasCorrection = [2] + +# Specify a bias correction for incoming long wave radiation flux. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# 2 - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# 3 - NCAR parametric GFS bias correction +LwBiasCorrection = [2] + +# Specify a bias correction for precipitation. +# 0 - No bias correction. +# 1 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +PrecipBiasCorrection = [0] + +[Downscaling] +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. + +# Specify a temperature downscaling method: +# 0 - No downscaling. +# 1 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation +# to the WRF-Hydro elevation. +# 2 - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +TemperatureDownscaling = [1] + +# Specify a surface pressure downscaling method: +# 0 - No downscaling. +# 1 - Use input elevation and WRF-Hydro elevation to downscale +# surface pressure. +PressureDownscaling = [1] + +# Specify a shortwave radiation downscaling routine. +# 0 - No downscaling +# 1 - Run a topographic adjustment using the WRF-Hydro elevation +ShortwaveDownscaling = [1] + +# Specify a precipitation downscaling routine. +# 0 - No downscaling +# 1 - NWM mountain mapper downscaling using monthly PRISM climo. +PrecipDownscaling = [0] + +# Specify a specific humidity downscaling routine. +# 0 - No downscaling +# 1 - Use regridded humidity, along with downscaled temperature/pressure +# to extrapolate a downscaled surface specific humidty. +HumidityDownscaling = [1] + +# Specify the input parameter directory containing necessary downscaling grids. +#DownscalingParamDirs = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +DownscalingParamDirs = /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + +[SuppForcing] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# 1 - MRMS GRIB2 hourly radar-only QPE +# 2 - MRMS GRIB2 hourly gage-corrected radar QPE +# 3 - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# 4 - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# 5 - CONUS MRMS MultiSensor Pass1 and Pass1 +# 6 - Hawaii MRMS MultiSensor Pass1 and Pass2 +# 7 - Alaska MRMS MultiSensor Pass1 and Pass2 +# 8 - Amir AK future option 1 +# 9 - Amir AK future option 2 +# 10 - Alaska MRMS (no liquid water fraction) +# 11 - Alaska Stage IV NWS Precip +SuppPcp = [10] + +# Specify the file type for each supplemental precipitation file (comma separated) +# Valid types are GRIB1, GRIB2, and NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +SuppPcpForcingTypes = [GRIB2] + +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +SuppPcpDirectories = /glade/p/ral/allral/zhangyx/Alaska.MRMS + +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# 0 - Not mandatory +# 1 - Mandatory +SuppPcpMandatory = [0] + +# Specify regridding options for the supplemental precipitation products. +RegridOptSuppPcp = [1] + +# Specify the time interpretation methods for the supplemental precipitation +# products. +SuppPcpTemporalInterpolation = [0] + +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +SuppPcpInputOffsets = [0] + +# Optional RQI method for radar-based data. +# 0 - Do not use any RQI filtering. Use all radar-based estimates. +# 1 - Use hourly MRMS Radar Quality Index grids. +# 2 - Use NWM monthly climatology grids (NWM only!!!!) +RqiMethod = 0 + +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +RqiThreshold = 0.9 + +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +#SuppPcpParamDir = /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/Alaska +SuppPcpParamDir = /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + +[Ensembles] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# FILL IN ENSEMBLE OPTIONS HERE..... +# Choose the CFS ensemble member number to process +cfsEnsNumber = + +[Custom] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +custom_input_fcst_freq = [] diff --git a/Config/YAML/template_forcing_engine_AK_AnA_hourly.yaml b/Config/YAML/template_forcing_engine_AK_AnA_hourly.yaml new file mode 100644 index 0000000..cae926e --- /dev/null +++ b/Config/YAML/template_forcing_engine_AK_AnA_hourly.yaml @@ -0,0 +1,498 @@ +#Document start +--- +# WRF-Hydro Forcing Engine YAML Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files. +#Forecast Example +# +# Forecast: +# AnAFlag: True +# LookBack: 180 +# RefcstBDateProc: 202009152300 +# RefcstEDateProc: 202009160000 +# Frequency: 60 +# Shift: 0 +# +# Forecast['AnAFlag'] +# Specify if this is an Analysis and Assimilation run (AnA). +# True - AnA run +# False +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +# +#Forecast['LookBack'] +# ONLY for realtime forecasting. +# Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +# +#Forecast['RefcstBDateProc'] +#Forecast['RefcstEDateProc'] +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +# +#Forecast['Frequency'] +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. = +# +#Forecast['Shift'] +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates. +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '202206052300' + RefcstEDateProc: '202206060000' + Shift: 0 + +#Geospatial Example +# +# Geospatial: +# GeogridIn: /glade/p/cisl/nwc/nwmv20_finals/CONUS/DOMAIN/geo_em.d01.conus_1km_NWMv2.0.nc +# SpatialMetaIn: /glade/p/cisl/nwc/nwmv20_finals/CONUS/DOMAIN/GEOGRID_LDASOUT_Spatial_Metadata_1km_NWMv2.0.nc +# +# Geospatial['GeogridIn'] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +# +# Geospatial['SpatialMetaIn'] +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file. +Geospatial: + GeogridIn: /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/geo_em.d02.nc + SpatialMetaIn: /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/GEOGRID_LDASOUT_Spatial_Metadata.nc + +#Input Example With Two Separate Inputs (a YAML list of input parameters) +# +#Input: +# - Forcing: RAP +# Type: GRIB2 +# Dir: /glade/p/ral/allral/zhangyx/RAP_Conus +# Mandatory: True +# Horizon: 60 +# Offset: 0 +# IgnoredBorderWidths: 0 +# RegriddingOpt: ESMF_BILINEAR +# TemporalInterp: NONE +## Custom: +## input_fcst_freq: 60 +# BiasCorrection: +# Temperature: NONE +# Pressure: NONE +# Humidity: NONE +# Wind: NONE +# Shortwave: NONE +# Longwave: NONE +# Precip: NONE +# Downscaling: +# Temperature: LAPSE_PRE_CALC +# Pressure: ELEV +# Shortwave: ELEV +# Precip: NONE +# Humidity: REGRID_TEMP_PRESS +# ParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# - Forcing: HRRR +# Type: GRIB2 +# Dir: /glade/p/ral/allral/zhangyx/HRRR_Conus +# Mandatory: True +# Horizon: 60 +# Offset: 0 +# IgnoredBorderWidths: 5 +# RegriddingOpt: ESMF_BILINEAR +# TemporalInterp: NONE +# BiasCorrection: +# Temperature: HRRR +# Pressure: NONE +# Humidity: NONE +# Wind: HRRR +# Shortwave: CUSTOM +# Longwave: CUSTOM +# Precip: NONE +# Downscaling: +# Temperature: LAPSE_PRE_CALC +# Pressure: ELEV +# Shortwave: ELEV +# Precip: NONE +# Humidity: REGRID_TEMP_PRESS +# ParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# +#Input[i]['Forcing'] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. +# NLDAS - GRIB retrospective files +# NARR - GRIB retrospective files +# GFS_GLOBAL - GRIB2 Global production files on the full gaussian grid +# NAM_NEST_CONUS - Nest GRIB2 Conus production files +# HRRR - GRIB2 Conus production files +# RAP - GRIB2 Conus 13km production files +# CFS_V2 - 6-hourly GRIB2 Global production files +# WRF_NEST_HI - GRIB2 Hawaii nest files +# GFS_GLOBAL_25 - GRIB2 Global production files on 0.25 degree lat/lon grids. +# CUSTOM_1 - Custom NetCDF hourly forcing files +# CUSTOM_2 - NetCDF hourly forcing files +# CUSTOM_3 - NetCDF hourly forcing files +# NAM_NEST_HI - 3-km NAM Nest. +# NAM_NEST_PR - 3-km NAM Nest. +# NAM_NEST_AK - 3-km Alaska Nest +# NAM_NEST_HI_RAD - NAM_Nest_3km_Hawaii_Radiation-Only +# NAM_NEST_PR_RAD - NAM_Nest_3km_PuertoRico_Radiation-Only +# WRF_ARW_PR - GRIB2 PuertoRico +# +#Input[i]['Type'] +# Specify the file type for each forcing +# GRIB +# GRIB2 +# NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +# +#Input[i]['Dir'] +# Specify the input directories for each forcing product. +# +#Input[i]['Mandatory'] +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# False - Not mandatory +# True - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +# +#Input[i]['Horizon'] +#Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +# +#Input[i]['Offset'] +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +# +# Input[i]['IgnoredBorderWidths'] +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +# +# Input[i]['RegriddingOpt'] +# Choose regridding options for each input forcing files being used. Options available are: +# ESMF_BILINEAR +# ESMF_NEAREST_NEIGHBOR +# ESMF_CONSERVATIVE_BILINEAR +# +# Input[i]['TemporalInterp'] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# NONE - No temporal interpolation. +# NEAREST_NEIGHBOR - Nearest Neighbor +# LINEAR_WEIGHT_AVG - Linear weighted average +# +# (Optional) +# Input[i]['Custom']['input_fcst_freq'] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +# +# Choose bias correction options for each of the input forcing files. +# +# Input[i]['BiasCorrection']['Temperature'] +# Specify a temperature bias correction method. +# NONE - No bias correction +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# HRRR - NCAR parametric HRRR bias correction +# +# Input[i]['BiasCorrection]['Pressure'] +# Specify a surface pressure bias correction method. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# +# Input[i]['BiasCorrection']['Humidity'] +# Specify a specific humidity bias correction method. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# +# Input[i]['BiasCorrection']['Wind'] +# Specify a wind bias correction. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# HRRR - NCAR parametric HRRR bias correction +# +# Input[i]['BiasCorrection']['Shortwave'] +# Specify a bias correction for incoming short wave radiation flux. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +# +# Input[i]['BiasCorrection']['Longwave'] +# Specify a bias correction for incoming long wave radiation flux. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# +# Input[i]['BiasCorrection']['Precip'] +# Specify a bias correction for precipitation. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. +# +# Input[i]['Downscaling']['Temperature'] +# Specify a temperature downscaling method: +# NONE - No downscaling. +# LAPSE_675 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation to the WRF-Hydro elevation. +# LAPSE_PRE_CALC - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +# +# Input[i]['Downscaling']['Pressure'] +# Specify a surface pressure downscaling method: +# NONE - No downscaling. +# ELEV - Use input elevation and WRF-Hydro elevation to downscale surface pressure. +# +# Input[i]['Downscaling']['Shortwave'] +# Specify a shortwave radiation downscaling routine. +# NONE - No downscaling +# ELEV - Run a topographic adjustment using the WRF-Hydro elevation +# +# Input[i]['Downscaling']['Precip'] +# Specify a precipitation downscaling routine. +# NONE - No downscaling +# NWM_MM - NWM mountain mapper downscaling using monthly PRISM climo. +# +# Input[i]['Downscaling']['Humidity'] +# Specify a specific humidity downscaling routine. +# NONE - No downscaling +# REGRID_TEMP_PRESS - Use regridded humidity, along with downscaled temperature/pressure to extrapolate a downscaled surface specific humidty. +# +# Input[i]['Downscaling']['ParamDir'] +# Specify the input parameter directory containing necessary downscaling grids. +Input: +- BiasCorrection: + Humidity: NONE + Longwave: CUSTOM + Precip: NONE + Pressure: NONE + Shortwave: CUSTOM + Temperature: HRRR + Wind: HRRR + Dir: /glade/p/ral/allral/zhangyx/HRRR_Alaska + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: HRRR_AK + Horizon: 180 + IgnoredBorderWidths: 10 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + forcingInputModYaml: '' + outputVarAttrYaml: '' + suppPrecipModYaml: '' + +#Output Example +# +#Output: +# Frequency: 60 +# Dir: /glade/scratch/bpetzke/ForcingEngine/Test/AnA +# ScratchDir: /glade/scratch/bpetzke/ForcingEngine/Test/AnA +# FloatOutput: SCALE_OFFSET +# CompressOutput: False +# +#Output['Frequency'] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +# +#Output['Dir'] +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +# +#Output['ScratchDir'] +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +# +#Output['FloatOutput'] +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# SCALE_OFFSET - Use scale/offset encoding +# FLOAT - Use floating-point encoding +# +#Output['CompressOutput'] +# Flag to activate netCDF4 deflate compression in the output files. +# False - Deactivate compression +# True - Activate compression + +Output: + CompressOutput: false + Dir: /glade/p/ral/wsap/petzke/fe_ak/AnA/data + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: /glade/p/ral/wsap/petzke/fe_ak/AnA/data + +#Regridding Example +# +# Regridding: +# WeightsDir: /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles +Regridding: + WeightsDir: null + +#Retrospective Example +# +# Retrospective: +# Flag: False +# BDateProc: 202002040000 +# EDateProc: 202002050000 +# +#Retrospective['Flag'] +# Specify to process forcings in retrosective mode +# False - No +# True - Yes +# +#Retrospective['BDateProc'] +#Retrospective['EDateProc'] +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours. +Retrospective: + Flag: false + +# SuppForcing Example +# +#SuppForcing: +# - Pcp: MRMR_CONUS_MS +# PcpTypes: GRIB2 +# PcpDir: /glade/p/ral/allral/zhangyx/CONUS.MRMS +# PcpMandatory: False +# RegridOptPcp: 1 +# PcpTemporalInterpolation: 0 +# PcpInputOffsets: 0 +# RqiMethod: NWM +# RqiThreshold: 0.9 +# PcpParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# +# SuppForcing[i]['Pcp'] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# MRMS - MRMS GRIB2 hourly radar-only QPE +# MRMS_GAGE - MRMS GRIB2 hourly gage-corrected radar QPE +# WRF_ARW_HI - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# WRF_ARW_PR - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# MRMS_CONUS_MS - CONUS MRMS MultiSensor Pass1 and Pass1 +# MRMS_HI_MS - Hawaii MRMS MultiSensor Pass1 and Pass2 +# MRMS_SBCV2 - Liquid Water Fraction (netCDF only) +# +# SuppForcing[i]['PcpType'] +# Specify the file type for each supplemental precipitation file (comma separated) +# GRIB1 +# GRIB2 +# NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +# +# SupportForcing[i]['PcpDir'] +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +# +# SuppForcing[i]['PcpMandatory'] +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# False - Not mandatory +# True - Mandatory +# +# SuppForcing[i]['RegridOptPcp'] +# Specify regridding options for the supplemental precipitation products. +# 1 - +# 2 - +# 3 - +# +# SuppForcing[i]['PcpTemporalInterp'] +# Specify the time interpretation methods for the supplemental precipitation +# products. +# 0 - +# 1 - +# 2 - +# +#SuppForcing[i]['PcpInputOffsets'] +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +# +#SuppForcing[i]['RqiMethod'] +# Optional RQI method for radar-based data. +# NONE - Do not use any RQI filtering. Use all radar-based estimates. +# MRMS - Use hourly MRMS Radar Quality Index grids. +# NWM - Use NWM monthly climatology grids (NWM only!!!!) +# +#SuppForcing[i]['RqiThreshold'] +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +# +#SuppForcing[i]['PcpParamDir'] +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology + +SuppForcing: +- Pcp: AK_MRMS + PcpDir: /glade/p/ral/allral/zhangyx/Alaska.MRMS + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ba3d718 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# WrfHydroForcing +### Python forcing engine for WRF-Hydro + +The WRF-Hydro Forcing Engine is primarily used to prepare input forcing variables for WRF-Hydro model simulations. +It is an object-oriented design (OOD) framework designed specifically for the WRF-Hydro modeling system. The framework has been written in Python version 3 and relies heavily on a few external software packages to properly read in native forcing products, while performing spatial interpolations in a parallel environment, along with downscaling and bias corrections. +Some of the input forcing products include historical regional and global reanalysis products that can be used for retrospective WRF-Hydro studies. Other products include operational real-time numerical weather prediction (NWP) models that are run at the National Centers for Environmental Prediction (NCEP). These operational products are of interest to real-time forecasting and nowcasting applications of the WRF-Hydro modeling system. Other products include supplemental precipitation products that offer spatially distributed quantitative precipitation estimates from sensors such as doppler radar. The Multi-Radar/Multi-Sensor (MRMS) product from the National Severe Storms Laboratory (NSSL) is a good example of supplemental precipitation the user may wish to ingest into the WRF-Hydro modeling architecture in place of NWP estimated precipitation. Future development will continue to refine the list of NWP products, QPE products, or other forcing datasets of interest. + +The following figure shows the overall flow of the Forcing Engine software package, along with the key components that lead to the eventual formation of forcing files that are ingested into the WRF- Hydro modeling system. +![image](https://user-images.githubusercontent.com/36771676/225337722-8275de94-f6ee-41e3-81e9-09906cd7f7c3.png) + +The starting point for using this framework is preparation of config files. Sample configs are placed in the Config subdirectory. The earlier version of the code used JSON Configuration. The new code uses YAML configuration and an older version can be converted to the new version by using the following python utility placed under the Utils subdirectory. + +Example Usage: +```python +export PYTHONPATH=$PYTHONPATH:~/git/WrfHydroForcing/ +export PYTHONPATH=$PYTHONPATH:~/git/WrfHydroForcing/core +./config2yaml.py ../Test/template_forcing_engine_AnA_v2.config ../Test/template_forcing_engine_AnA_v2_example.yaml +``` +Example usage of running the framework: +```python +time mpiexec python3 -E ~/git/WrfHydroForcing/genForcing.py ~/git/WrfHydroForcing/Config/YAML/template_forcing_engine_PRVI_AnA.yaml 3.0 AnA +``` + +### System Level Requirements +1. The Forcing Engine code is written in Python, but it can only run on a Unix-based system. This is required as additional dependencies only run in this environment as well. For these reasons, the Forcing Engine cannot run in either Windows or Mac OS. The system will need to have Python 3 installed on the system. The current version of this software is not backwards compatible with Python 2. For larger WRF-Hydro modeling domains, it’s encouraged to make sure there is enough memory on the system to accommodate the interpolation from coarse grids to the higher resolution WRF-Hydro geogrid file. +2. The user is required to have an installation of MPI on the machine they will be executing the Forcing Engine on. For many users, this requirement may already be fulfilled if WRF-Hydro is being ran in parallel using MPI as well. The Python code uses MPI-wrappers to split the processing amongst multiple processors. Even if the user specifies an execution on one processor, MPI software is still being invoked. +3. Installation of Earth System Modeling Framework (ESMF) Software and ESMPy. +4. Installation of wgrib2. diff --git a/Test/AK/ana/template_forcing_engine_AK_AnA_hourly.yaml b/Test/AK/ana/template_forcing_engine_AK_AnA_hourly.yaml new file mode 100644 index 0000000..09d700e --- /dev/null +++ b/Test/AK/ana/template_forcing_engine_AK_AnA_hourly.yaml @@ -0,0 +1,63 @@ +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '202206052300' + RefcstEDateProc: '202206060000' + Shift: 0 + +Geospatial: + GeogridIn: /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/geo_em.d02.nc + SpatialMetaIn: /glade/p/cisl/nwc/zhangyx/AlaskaNWM/DOMAIN/GEOGRID_LDASOUT_Spatial_Metadata.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: CUSTOM + Precip: NONE + Pressure: NONE + Shortwave: CUSTOM + Temperature: HRRR + Wind: HRRR + Dir: /glade/p/ral/allral/zhangyx/HRRR_Alaska + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: HRRR_AK + Horizon: 180 + IgnoredBorderWidths: 10 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: /glade/p/ral/wsap/petzke/fe_ak/AnA/data + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: /glade/p/ral/wsap/petzke/fe_ak/AnA/data + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: AK_MRMS + PcpDir: /glade/p/ral/allral/zhangyx/Alaska.MRMS + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: /glade/p/cisl/nwc/zhangyx/NWM_v21_Params/Alaska + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: + - 0.9 diff --git a/Util/config2yaml.py b/Util/config2yaml.py new file mode 100755 index 0000000..06fa26c --- /dev/null +++ b/Util/config2yaml.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import argparse +import sys +from pprint import pprint +import configparser + +import numpy as np +import yaml +import enumConfig +import config +import config_v1 +import yaml_comment_template as templ + +""" +Script to convert old-style WrfHydroForcing .config files to new-style .yaml + +Example Usage: + +export PYTHONPATH=$PYTHONPATH:~/git/WrfHydroForcing/ +export PYTHONPATH=$PYTHONPATH:~/git/WrfHydroForcing/core +./config2yaml.py ../Test/template_forcing_engine_AnA_v2.config ../Test/template_forcing_engine_AnA_v2_example.yaml +""" + +class SpaceDumper(yaml.SafeDumper): + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + +def convert(config_file, yaml_file): + print('Converting %s -> %s' % (config_file,yaml_file)) + + config_old = config_v1.ConfigOptions(config_file) + config_old.read_config() + config_params = vars(config_old) + #pprint(config_params) + + out_yaml = {'Input':[],'Output':{},'YamlConfig':{},'Retrospective':{},'Forecast':{},'Geospatial':{},'Regridding':{}, 'SuppForcing':[]} + custom_count = 0 + for i in range(len(config_params['input_forcings'])): + input_dict = {} + input_dict['Forcing'] = enumConfig.ForcingEnum(config_params['input_forcings'][i]).name + input_dict['Type'] = config_params['input_force_types'][i] + input_dict['Dir'] = config_params['input_force_dirs'][i] + input_dict['Mandatory'] = bool(config_params['input_force_mandatory'][i]) + input_dict['Horizon'] = config_params['fcst_input_horizons'][i] + input_dict['Offset'] = config_params['fcst_input_offsets'][i] + input_dict['IgnoredBorderWidths'] = config_params['ignored_border_widths'][i] + input_dict['RegriddingOpt'] = enumConfig.RegriddingOptEnum(config_params['regrid_opt'][i]).name + input_dict['TemporalInterp'] = enumConfig.TemporalInterpEnum(config_params['forceTemoralInterp'][i]).name + if input_dict['Forcing'] == 'CUSTOM_1': + input_dict['Custom'] = {'input_fcst_freq':config_params['customFcstFreq'][custom_count]} + custom_count += 1 + input_dict['BiasCorrection'] = {} + input_dict['BiasCorrection']['Temperature'] = enumConfig.BiasCorrTempEnum(config_params['t2BiasCorrectOpt'][i]).name + input_dict['BiasCorrection']['Pressure'] = enumConfig.BiasCorrPressEnum(config_params['psfcBiasCorrectOpt'][i]).name + input_dict['BiasCorrection']['Humidity'] = enumConfig.BiasCorrHumidEnum(config_params['q2BiasCorrectOpt'][i]).name + input_dict['BiasCorrection']['Wind'] = enumConfig.BiasCorrWindEnum(config_params['windBiasCorrect'][i]).name + input_dict['BiasCorrection']['Shortwave'] = enumConfig.BiasCorrSwEnum(config_params['swBiasCorrectOpt'][i]).name + input_dict['BiasCorrection']['Longwave'] = enumConfig.BiasCorrLwEnum(config_params['lwBiasCorrectOpt'][i]).name + input_dict['BiasCorrection']['Precip'] = enumConfig.BiasCorrPrecipEnum(config_params['precipBiasCorrectOpt'][i]).name + input_dict['Downscaling'] = {} + input_dict['Downscaling']['Temperature'] = enumConfig.DownScaleTempEnum(config_params['t2dDownscaleOpt'][i]).name + input_dict['Downscaling']['Pressure'] = enumConfig.DownScalePressEnum(config_params['psfcDownscaleOpt'][i]).name + input_dict['Downscaling']['Shortwave'] = enumConfig.DownScaleSwEnum(config_params['swDownscaleOpt'][i]).name + input_dict['Downscaling']['Precip'] = enumConfig.DownScalePrecipEnum(config_params['precipDownscaleOpt'][i]).name + input_dict['Downscaling']['Humidity'] = enumConfig.DownScaleHumidEnum(config_params['q2dDownscaleOpt'][i]).name + input_dict['Downscaling']['ParamDir'] = config_params['dScaleParamDirs'][i] + if input_dict['Forcing'] == 'CFS_V2': + input_dict['Ensembles'] = {'cfsEnsNumber':config_params['cfsv2EnsMember']} + out_yaml['Input'].append(input_dict) + + out_yaml['Output']['Frequency'] = config_params['output_freq'] + out_yaml['Output']['Dir'] = config_params['output_dir'] + out_yaml['Output']['ScratchDir'] = config_params['scratch_dir'] + out_yaml['Output']['CompressOutput'] = bool(config_params['useCompression']) + out_yaml['Output']['FloatOutput'] = enumConfig.OutputFloatEnum(config_params['useFloats']).name + out_yaml['YamlConfig']['forcingInputModYaml'] = '' + out_yaml['YamlConfig']['suppPrecipModYaml'] = '' + out_yaml['YamlConfig']['outputVarAttrYaml'] = '' + out_yaml['Retrospective']['Flag'] = bool(config_params['retro_flag']) + + #Due to internal config_v1.py logic that modifies b_date_proc and e_date_proc + configpsr = configparser.ConfigParser() + configpsr.read(config_file) + + if out_yaml['Retrospective']['Flag']: + out_yaml['Retrospective']['BDateProc'] = configparser['Retrospective']['BDateProc'] + out_yaml['Retrospective']['EDateProc'] = configparser['Retrospective']['EDateProc'] + + out_yaml['Forecast']['AnAFlag'] = bool(config_params['ana_flag']) + out_yaml['Forecast']['LookBack'] = config_params['look_back'] + out_yaml['Forecast']['RefcstBDateProc'] = configpsr['Forecast']['RefcstBDateProc'] + out_yaml['Forecast']['RefcstEDateProc'] = configpsr['Forecast']['RefcstEDateProc'] + out_yaml['Forecast']['Frequency'] = config_params['fcst_freq'] + out_yaml['Forecast']['Shift'] = config_params['fcst_shift'] + + out_yaml['Geospatial']['GeogridIn'] = config_params['geogrid'] + out_yaml['Geospatial']['SpatialMetaIn'] = config_params['spatial_meta'] + + out_yaml['Regridding']['WeightsDir'] = config_params['weightsDir'] + + for i in range(len(config_params['supp_precip_forcings'])): + supp_forcing_dict = {} + supp_forcing_dict['Pcp'] = enumConfig.SuppForcingPcpEnum(config_params['supp_precip_forcings'][i]).name + supp_forcing_dict['PcpType'] = config_params['supp_precip_file_types'][i] + supp_forcing_dict['PcpDir'] = config_params['supp_precip_dirs'][i] + supp_forcing_dict['PcpMandatory'] = bool(config_params['supp_precip_mandatory'][i]) + supp_forcing_dict['RegridOptPcp'] = config_params['regrid_opt_supp_pcp'][i] + supp_forcing_dict['PcpTemporalInterp'] = config_params['suppTemporalInterp'][i] + supp_forcing_dict['PcpInputOffsets'] = config_params['supp_input_offsets'][i] + if config_params['rqiMethod']: + supp_forcing_dict['RqiMethod'] = enumConfig.SuppForcingRqiMethodEnum(config_params['rqiMethod']).name + else: + supp_forcing_dict['RqiMethod'] = "NONE" + if config_params['rqiThresh']: + supp_forcing_dict['RqiThreshold'] = config_params['rqiThresh'] + supp_forcing_dict['PcpParamDir'] = config_params['supp_precip_param_dir'] + out_yaml['SuppForcing'].append(supp_forcing_dict) + + + #print(yaml.dump(out_yaml,Dumper=SpaceDumper,default_flow_style=False)) + with open(yaml_file,'w') as f: + yaml.dump(out_yaml,f,Dumper=SpaceDumper,default_flow_style=False) + +def add_comments(yaml_file): + lines = [] + with open(yaml_file) as f: + lines.append(templ.comments['Header']) + for line in f.readlines(): + line = line.rstrip() + if line in {"Input:", "Output:", "Retrospective:", "Forecast:", "Geospatial:", "Regridding:", "SuppForcing:", "Ensembles:"}: + lines.append(templ.comments[line[:-1]]) + lines.append(line) + + #print("\n".join(lines)) + with open(yaml_file, "w") as f: + f.write("\n".join(lines)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('config_file',type=str,help='Input old-style .conf file') + parser.add_argument('yaml_file',type=str,help='Output new-style .yaml file') + args = parser.parse_args() + convert(args.config_file,args.yaml_file) + add_comments(args.yaml_file) + + +if __name__ == '__main__': + main() diff --git a/Util/yaml_comment_template.py b/Util/yaml_comment_template.py new file mode 100644 index 0000000..724e48c --- /dev/null +++ b/Util/yaml_comment_template.py @@ -0,0 +1,456 @@ +#Module provides a template of yaml comment headers to insert for each section of +#auto-generated yaml config + +comments = { +'Header' : +"""#Document start +--- +# WRF-Hydro Forcing Engine YAML Configuration File +# +# Input options to the forcing engine include: +# 1.) Choices for input forcing files to use. +# 2.) Options for specifying date ranges and forecast intervals +# for input files. +# 3.) Choices for ESMF regridding techniques. +# 4.) Choices for optional downscaling techniques. +# 5.) Choices for optional bias correction techniques. +# 6.) Choices for optional supplemental precipitation products. +# 7.) Choices for optional ensemble member variations. +# 8.) Choices for output directories to place final output files.""", +'Input' : +"""#Input Example With Two Separate Inputs (a YAML list of input parameters) +# +#Input: +# - Forcing: RAP +# Type: GRIB2 +# Dir: /glade/p/ral/allral/zhangyx/RAP_Conus +# Mandatory: True +# Horizon: 60 +# Offset: 0 +# IgnoredBorderWidths: 0 +# RegriddingOpt: ESMF_BILINEAR +# TemporalInterp: NONE +## Custom: +## input_fcst_freq: 60 +# BiasCorrection: +# Temperature: NONE +# Pressure: NONE +# Humidity: NONE +# Wind: NONE +# Shortwave: NONE +# Longwave: NONE +# Precip: NONE +# Downscaling: +# Temperature: LAPSE_PRE_CALC +# Pressure: ELEV +# Shortwave: ELEV +# Precip: NONE +# Humidity: REGRID_TEMP_PRESS +# ParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# - Forcing: HRRR +# Type: GRIB2 +# Dir: /glade/p/ral/allral/zhangyx/HRRR_Conus +# Mandatory: True +# Horizon: 60 +# Offset: 0 +# IgnoredBorderWidths: 5 +# RegriddingOpt: ESMF_BILINEAR +# TemporalInterp: NONE +# BiasCorrection: +# Temperature: HRRR +# Pressure: NONE +# Humidity: NONE +# Wind: HRRR +# Shortwave: CUSTOM +# Longwave: CUSTOM +# Precip: NONE +# Downscaling: +# Temperature: LAPSE_PRE_CALC +# Pressure: ELEV +# Shortwave: ELEV +# Precip: NONE +# Humidity: REGRID_TEMP_PRESS +# ParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# +#Input[i]['Forcing'] +# Choose a set of value(s) of forcing variables to be processed for +# WRF-Hydro. Please be advised that the order of which the values are +# chosen below are the order that the final products will be layered +# into the final LDASIN files. See documentation for additional +# information and examples. +# The following is a global set of key values to map forcing files +# to variables within LDASIN files for WRF-Hydro. The forcing engine +# will map files to external variable names internally. For custom +# external native forcing files (see documenation), the code will +# expect a set of named variables to process. +# NLDAS - GRIB retrospective files +# NARR - GRIB retrospective files +# GFS_GLOBAL - GRIB2 Global production files on the full gaussian grid +# NAM_NEST_CONUS - Nest GRIB2 Conus production files +# HRRR - GRIB2 Conus production files +# RAP - GRIB2 Conus 13km production files +# CFS_V2 - 6-hourly GRIB2 Global production files +# WRF_NEST_HI - GRIB2 Hawaii nest files +# GFS_GLOBAL_25 - GRIB2 Global production files on 0.25 degree lat/lon grids. +# CUSTOM_1 - Custom NetCDF hourly forcing files +# CUSTOM_2 - NetCDF hourly forcing files +# CUSTOM_3 - NetCDF hourly forcing files +# NAM_NEST_HI - 3-km NAM Nest. +# NAM_NEST_PR - 3-km NAM Nest. +# NAM_NEST_AK - 3-km Alaska Nest +# NAM_NEST_HI_RAD - NAM_Nest_3km_Hawaii_Radiation-Only +# NAM_NEST_PR_RAD - NAM_Nest_3km_PuertoRico_Radiation-Only +# WRF_ARW_PR - GRIB2 PuertoRico +# +#Input[i]['Type'] +# Specify the file type for each forcing +# GRIB +# GRIB2 +# NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +# +#Input[i]['Dir'] +# Specify the input directories for each forcing product. +# +#Input[i]['Mandatory'] +# Specify whether the input forcings listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# False - Not mandatory +# True - Mandatory +# NOTE!!! If not files are found for any products, code will error out indicating +# the final field is all missing values. +# +#Input[i]['Horizon'] +#Specify how much (in minutes) of each input forcing is desires for each +# forecast cycle. See documentation for examples. The length of +# this array must match the input forcing choices. +# +#Input[i]['Offset'] +# This option is for applying an offset to input forcings to use a different +# forecasted interval. For example, a user may wish to use 4-5 hour forecasted +# fields from an NWP grid from one of their input forcings. In that instance +# the offset would be 4 hours, but 0 for other remaining forcings. +# +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +# +# Input[i]['IgnoredBorderWidths'] +# Specify a border width (in grid cells) to ignore for each input dataset. +# NOTE: generally, the first input forcing should always be zero or there will be missing data in the final output +# +# Input[i]['RegriddingOpt'] +# Choose regridding options for each input forcing files being used. Options available are: +# ESMF_BILINEAR +# ESMF_NEAREST_NEIGHBOR +# ESMF_CONSERVATIVE_BILINEAR +# +# Input[i]['TemporalInterp'] +# Specify an temporal interpolation for the forcing variables. +# Interpolation will be done between the two neighboring +# input forcing states that exist. If only one nearest +# state exist (I.E. only a state forward in time, or behind), +# then that state will be used as a "nearest neighbor". +# NOTE - All input options here must be of the same length +# of the input forcing number. Also note all temporal interpolation +# occurs BEFORE downscaling and bias correction. +# NONE - No temporal interpolation. +# NEAREST_NEIGHBOR - Nearest Neighbor +# LINEAR_WEIGHT_AVG - Linear weighted average +# +# (Optional) +# Input[i]['Custom']['input_fcst_freq'] +# These are options for specifying custom input NetCDF forcing files (in minutes). +# Choose the input frequency of files that are being processed. I.E., are the +# input files every 15 minutes, 60 minutes, 3-hours, etc. Please specify the +# length of custom input frequencies to match the number of custom NetCDF inputs +# selected above in the Logistics section. +# +# Choose bias correction options for each of the input forcing files. +# +# Input[i]['BiasCorrection']['Temperature'] +# Specify a temperature bias correction method. +# NONE - No bias correction +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# HRRR - NCAR parametric HRRR bias correction +# +# Input[i]['BiasCorrection]['Pressure'] +# Specify a surface pressure bias correction method. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# +# Input[i]['BiasCorrection']['Humidity'] +# Specify a specific humidity bias correction method. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# +# Input[i]['BiasCorrection']['Wind'] +# Specify a wind bias correction. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis - based on hour of day (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# HRRR - NCAR parametric HRRR bias correction +# +# Input[i]['BiasCorrection']['Shortwave'] +# Specify a bias correction for incoming short wave radiation flux. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis (USE WITH CAUTION). +# +# Input[i]['BiasCorrection']['Longwave'] +# Specify a bias correction for incoming long wave radiation flux. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# CUSTOM - Custom NCAR bias-correction based on HRRRv3 analysis, blanket adjustment (USE WITH CAUTION). +# GFS - NCAR parametric GFS bias correction +# +# Input[i]['BiasCorrection']['Precip'] +# Specify a bias correction for precipitation. +# NONE - No bias correction. +# CFS_V2 - CFSv2 - NLDAS2 Parametric Distribution - NWM ONLY +# +# Choose downscaling options for each of the input forcing files. Length of each option +# must match the length of input forcings. +# +# Input[i]['Downscaling']['Temperature'] +# Specify a temperature downscaling method: +# NONE - No downscaling. +# LAPSE_675 - Use a simple lapse rate of 6.75 degrees Celsius to get from the model elevation to the WRF-Hydro elevation. +# LAPSE_PRE_CALC - Use a pre-calculated lapse rate regridded to the WRF-Hydro domain. +# +# Input[i]['Downscaling']['Pressure'] +# Specify a surface pressure downscaling method: +# NONE - No downscaling. +# ELEV - Use input elevation and WRF-Hydro elevation to downscale surface pressure. +# +# Input[i]['Downscaling']['Shortwave'] +# Specify a shortwave radiation downscaling routine. +# NONE - No downscaling +# ELEV - Run a topographic adjustment using the WRF-Hydro elevation +# +# Input[i]['Downscaling']['Precip'] +# Specify a precipitation downscaling routine. +# NONE - No downscaling +# NWM_MM - NWM mountain mapper downscaling using monthly PRISM climo. +# +# Input[i]['Downscaling']['Humidity'] +# Specify a specific humidity downscaling routine. +# NONE - No downscaling +# REGRID_TEMP_PRESS - Use regridded humidity, along with downscaled temperature/pressure to extrapolate a downscaled surface specific humidty. +# +# Input[i]['Downscaling']['ParamDir'] +# Specify the input parameter directory containing necessary downscaling grids.""", +'Output' : +"""#Output Example +# +#Output: +# Frequency: 60 +# Dir: /glade/scratch/bpetzke/ForcingEngine/Test/AnA +# ScratchDir: /glade/scratch/bpetzke/ForcingEngine/Test/AnA +# FloatOutput: SCALE_OFFSET +# CompressOutput: False +# +#Output['Frequency'] +# Specify the output frequency in minutes. +# Note that any frequencies at higher intervals +# than what is provided as input will entail input +# forcing data being temporally interpolated. +# +#Output['Dir'] +# Specify a top level output directory. For re-forecasts +# and forecasts, sub-directories for each forecast cycle +# will be generated. For retrospective processing, final +# output files will be placed in this directory. +# +#Output['ScratchDir'] +# Specify a scratch directory that will be used +# for storage of temporary files. These files +# will be removed automatically by the program. +# +#Output['FloatOutput'] +# Flag to use floating point output vs scale_factor / add_offset byte packing in +# the output files (the default) +# SCALE_OFFSET - Use scale/offset encoding +# FLOAT - Use floating-point encoding +# +#Output['CompressOutput'] +# Flag to activate netCDF4 deflate compression in the output files. +# False - Deactivate compression +# True - Activate compression +""", +'Retrospective' : +"""#Retrospective Example +# +# Retrospective: +# Flag: False +# BDateProc: 202002040000 +# EDateProc: 202002050000 +# +#Retrospective['Flag'] +# Specify to process forcings in retrosective mode +# False - No +# True - Yes +# +#Retrospective['BDateProc'] +#Retrospective['EDateProc'] +# Choose the beginning date of processing forcing files. +# NOTE - Dates are given in YYYYMMDDHHMM format +# If in real-time forecasting mode, leave as -9999. +# These dates get over-ridden in lookBackHours.""", +'Forecast' : +"""#Forecast Example +# +# Forecast: +# AnAFlag: True +# LookBack: 180 +# RefcstBDateProc: 202009152300 +# RefcstEDateProc: 202009160000 +# Frequency: 60 +# Shift: 0 +# +# Forecast['AnAFlag'] +# Specify if this is an Analysis and Assimilation run (AnA). +# True - AnA run +# False +# Setting this flag will change the behavior of some Bias Correction routines as wel +# as the ForecastInputOffsets options (see below for more information) +# +#Forecast['LookBack'] +# ONLY for realtime forecasting. +# Specify a lookback period in minutes to process data. +# This overrides any BDateProc/EDateProc options passed above. +# If no LookBack specified, please specify -9999. +# +#Forecast['RefcstBDateProc'] +#Forecast['RefcstEDateProc'] +# If running reforecasts, specify a window below. This will override +# using the LookBack value to calculate a processing window. +# +#Forecast['Frequency'] +# Specify a forecast frequency in minutes. This value specifies how often +# to generate a set of forecast forcings. If generating hourly retrospective +# forcings, specify this value to be 60. = +# +#Forecast['Shift'] +# Forecast cycles are determined by splitting up a day by equal +# ForecastFrequency interval. If there is a desire to shift the +# cycles to a different time step, ForecastShift will shift forecast +# cycles ahead by a determined set of minutes. For example, ForecastFrequency +# of 6 hours will produce forecasts cycles at 00, 06, 12, and 18 UTC. However, +# a ForecastShift of 1 hour will produce forecast cycles at 01, 07, +# 13, and 18 UTC. NOTE - This is only used by the realtime instance +# to calculate forecast cycles accordingly. Re-forecasts will use the beginning +# and ending dates specified in conjunction with the forecast frequency +# to determine forecast cycle dates.""", +'Geospatial' : +"""#Geospatial Example +# +# Geospatial: +# GeogridIn: /glade/p/cisl/nwc/nwmv20_finals/CONUS/DOMAIN/geo_em.d01.conus_1km_NWMv2.0.nc +# SpatialMetaIn: /glade/p/cisl/nwc/nwmv20_finals/CONUS/DOMAIN/GEOGRID_LDASOUT_Spatial_Metadata_1km_NWMv2.0.nc +# +# Geospatial['GeogridIn'] +# Specify a geogrid file that defines the WRF-Hydro (or NWM) domain to which +# the forcings are being processed to. +# +# Geospatial['SpatialMetaIn'] +# Specify the optional land spatial metadata file. If found, coordinate projection information +# and coordinate will be translated from to the final output file.""", +'Regridding' : +"""#Regridding Example +# +# Regridding: +# WeightsDir: /glade/p/cisl/nwc/nwm_forcings/ESMFWeightFiles""", +'SuppForcing' : +"""# SuppForcing Example +# +#SuppForcing: +# - Pcp: MRMR_CONUS_MS +# PcpTypes: GRIB2 +# PcpDir: /glade/p/ral/allral/zhangyx/CONUS.MRMS +# PcpMandatory: False +# RegridOptPcp: 1 +# PcpTemporalInterpolation: 0 +# PcpInputOffsets: 0 +# RqiMethod: NWM +# RqiThreshold: 0.9 +# PcpParamDir: /glade/p/cisl/nwc/nwm_forcings/NWM_v21_Params/AnA +# +# SuppForcing[i]['Pcp'] +# Choose a set of supplemental precipitation file(s) to layer +# into the final LDASIN forcing files processed from +# the options above. The following is a mapping of +# numeric values to external input native forcing files: +# MRMS - MRMS GRIB2 hourly radar-only QPE +# MRMS_GAGE - MRMS GRIB2 hourly gage-corrected radar QPE +# WRF_ARW_HI - WRF-ARW 2.5 km 48-hr Hawaii nest precipitation. +# WRF_ARW_PR - WRF-ARW 2.5 km 48-hr Puerto Rico nest precipitation. +# MRMS_CONUS_MS - CONUS MRMS MultiSensor Pass1 and Pass1 +# MRMS_HI_MS - Hawaii MRMS MultiSensor Pass1 and Pass2 +# MRMS_SBCV2 - Liquid Water Fraction (netCDF only) +# +# SuppForcing[i]['PcpType'] +# Specify the file type for each supplemental precipitation file (comma separated) +# GRIB1 +# GRIB2 +# NETCDF +# (GRIB files will be converted internally with WGRIB[2]) +# +# SupportForcing[i]['PcpDir'] +# Specify the correponding supplemental precipitation directories +# that will be searched for input files. +# +# SuppForcing[i]['PcpMandatory'] +# Specify whether the Supplemental Precips listed above are mandatory, or optional. +# This is important for layering contingencies if a product is missing, +# but forcing files are still desired. +# False - Not mandatory +# True - Mandatory +# +# SuppForcing[i]['RegridOptPcp'] +# Specify regridding options for the supplemental precipitation products. +# 1 - +# 2 - +# 3 - +# +# SuppForcing[i]['PcpTemporalInterp'] +# Specify the time interpretation methods for the supplemental precipitation +# products. +# 0 - +# 1 - +# 2 - +# +#SuppForcing[i]['PcpInputOffsets'] +# In AnA runs, this value is the offset from the available forecast and 00z +# For example, if forecast are available at 06z and 18z, set this value to 6 +# +#SuppForcing[i]['RqiMethod'] +# Optional RQI method for radar-based data. +# NONE - Do not use any RQI filtering. Use all radar-based estimates. +# MRMS - Use hourly MRMS Radar Quality Index grids. +# NWM - Use NWM monthly climatology grids (NWM only!!!!) +# +#SuppForcing[i]['RqiThreshold'] +# Optional RQI threshold to be used to mask out. Currently used for MRMS products. +# Please choose a value from 0.0-1.0. Associated radar quality index files will be expected +# from MRMS data. +# +#SuppForcing[i]['PcpParamDir'] +# Specify an optional directory that contains supplemental precipitation parameter fields, +# I.E monthly RQI climatology +""", +'Ensembles' : +"""# Ensembles Example +# +# Ensembles: +# cfsEnsNumber: [] +# +# Ensembles['cfsEnsNumber'][i] +# Choose ensemble options for each input forcing file being used. Ensemble options include: +# 1, 2, 3, 4 +# Choose the CFS ensemble member number to process""" +} diff --git a/core/Regridding/__init__.py b/core/Regridding/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/Regridding/regrid_base.py b/core/Regridding/regrid_base.py new file mode 100644 index 0000000..d7e2442 --- /dev/null +++ b/core/Regridding/regrid_base.py @@ -0,0 +1,314 @@ +import os +import time +from abc import ABC, abstractmethod +from contextlib import contextmanager + +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + +import numpy as np + +from core import err_handler + + +class Regridder(ABC): + def __init__(self, name:str, config_options, wrf_hydro_geo_meta, mpi_config): + self.name = name + self.config = config_options + self.geo_meta = wrf_hydro_geo_meta + self.mpi = mpi_config + self.IS_MPI_ROOT = mpi_config.rank == 0 + + self._next_file_number = 0 + + ''' + Abstract methods overridden in subclasses: + ''' + + @abstractmethod + def regrid_input(self, input_forcings): + pass + + ''' + Utility methods: + ''' + + def mkfilename(self): + self._next_file_number += 1 + return '{}'.format(self._next_file_number) + + @contextmanager + def parallel_block(self): + yield + err_handler.check_program_status(self.config, self.mpi) + + def log_critical(self, msg, root_only=False): + if not root_only or self.IS_MPI_ROOT: + # TODO: rewrite this with cleaned-up OO err_handler + self.config.errMsg = msg + err_handler.log_critical(self.config, self.mpi) + + def log_status(self, msg, root_only=False): + if not root_only or self.IS_MPI_ROOT: + # TODO: rewrite this with cleaned-up OO err_handler + self.config.statusMsg = msg + err_handler.log_msg(self.config, self.mpi) + + def calculate_weights(self, dataset, forcings, forcing_idx): + """ + Function to calculate ESMF weights based on the output ESMF + field previously calculated, along with input lat/lon grids, + and a sample dataset. + :param dataset: + :param forcings: + :param forcing_idx: + """ + # TODO: should this be moved into the InputForcing object? + + with self.parallel_block(): + if self.IS_MPI_ROOT: + try: + forcings.ny_global = dataset.variables[forcings.netcdf_var_names[forcing_idx]].shape[1] + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to extract Y shape size from: " + + forcings.netcdf_var_names[forcing_idx] + " from: " + + forcings.tmpFile + " (" + str(err) + ")") + try: + forcings.nx_global = dataset.variables[forcings.netcdf_var_names[forcing_idx]].shape[2] + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to extract X shape size from: " + + forcings.netcdf_var_names[forcing_idx] + " from: " + + forcings.tmpFile + " (" + str(err) + ")") + + # Broadcast the forcing nx/ny values + forcings.ny_global = self.mpi.broadcast_parameter(forcings.ny_global, self.config) + forcings.nx_global = self.mpi.broadcast_parameter(forcings.nx_global, self.config) + + with self.parallel_block(): + try: + # noinspection PyTypeChecker + forcings.esmf_grid_in = ESMF.Grid(np.array([forcings.ny_global, forcings.nx_global]), + staggerloc=ESMF.StaggerLoc.CENTER, + coord_sys=ESMF.CoordSys.SPH_DEG) + + forcings.x_lower_bound = forcings.esmf_grid_in.lower_bounds[ESMF.StaggerLoc.CENTER][1] + forcings.x_upper_bound = forcings.esmf_grid_in.upper_bounds[ESMF.StaggerLoc.CENTER][1] + forcings.y_lower_bound = forcings.esmf_grid_in.lower_bounds[ESMF.StaggerLoc.CENTER][0] + forcings.y_upper_bound = forcings.esmf_grid_in.upper_bounds[ESMF.StaggerLoc.CENTER][0] + forcings.nx_local = forcings.x_upper_bound - forcings.x_lower_bound + forcings.ny_local = forcings.y_upper_bound - forcings.y_lower_bound + + # Check to make sure we have enough dimensionality to run regridding. + # ESMF requires both grids to have a size of at least 2. + if forcings.nx_local < 2 or forcings.ny_local < 2: + self.log_critical("You have either specified too many cores for: {}," + + "or your input forcing grid is too small to process. " + + "Local grid must have x/y dimension size of 2.".format(forcings.productName)) + + except ESMF.ESMPyException as esmf_error: + self.log_critical("Unable to create source ESMF grid from " + + "temporary file: {} ({})".format(forcings.tmpFile, esmf_error)) + + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to extract local X/Y boundaries from global grid from " + + "temporary file: {} ({})".format(forcings.tmpFile, err)) + + with self.parallel_block(): + if self.IS_MPI_ROOT: + # Process lat/lon values from the GFS grid. + if len(dataset.variables['latitude'].shape) == 3: + # We have 3D grids already in place. + global_latitude_array = dataset.variables['latitude'][0, :, :] + global_longitude_array = dataset.variables['longitude'][0, :, :] + elif len(dataset.variables['longitude'].shape) == 2: + # We have 2D grids already in place. + global_latitude_array = dataset.variables['latitude'][:, :] + global_longitude_array = dataset.variables['longitude'][:, :] + elif len(dataset.variables['latitude'].shape) == 1: + # We have 1D lat/lons we need to translate into 2D grids. + global_latitude_array = np.repeat(dataset.variables['latitude'][:][:, np.newaxis], + forcings.nx_global, axis=1) + global_longitude_array = np.tile(dataset.variables['longitude'][:], (forcings.ny_global, 1)) + else: + global_latitude_array = global_longitude_array = None + + # Scatter global GFS latitude grid to processors.. + local_latitude_array = self.mpi.scatter_array(forcings, global_latitude_array, self.config) + local_longitude_array = self.mpi.scatter_array(forcings, global_longitude_array, self.config) + + with self.parallel_block(): + try: + forcings.esmf_lats = forcings.esmf_grid_in.get_coords(1) + except ESMF.GridException as ge: + self.log_critical("Unable to locate latitude coordinate object within input ESMF grid: " + str(ge)) + + try: + forcings.esmf_lons = forcings.esmf_grid_in.get_coords(0) + except ESMF.GridException as ge: + self.log_critical("Unable to locate longitude coordinate object within input ESMF grid: " + str(ge)) + + forcings.esmf_lats[:, :] = local_latitude_array + forcings.esmf_lons[:, :] = local_longitude_array + # del local_latitude_array + # del local_longitude_array + del global_latitude_array + del global_longitude_array + + # Create a ESMF field to hold the incoming data. + with self.parallel_block(): + try: + forcings.esmf_field_in = ESMF.Field(forcings.esmf_grid_in, name=forcings.productName + "_NATIVE") + + # Scatter global grid to processors.. + if self.IS_MPI_ROOT: + var_tmp_global = dataset[forcings.netcdf_var_names[forcing_idx]][0, :, :] + # Set all valid values to 1.0, and all missing values to 0.0. This will + # be used to generate an output mask that is used later on in downscaling, layering, + # etc. + var_tmp_global[:, :] = 1.0 + else: + var_tmp_global = None + var_tmp_local = self.mpi.scatter_array(forcings, var_tmp_global, self.config) + except ESMF.ESMPyException as esmf_error: + self.log_critical("Unable to create ESMF field object: " + str(esmf_error)) + + # Place temporary data into the field array for generating the regridding object. + forcings.esmf_field_in.data[:, :] = var_tmp_local + + # ## CALCULATE WEIGHT ## # + # Try to find a pre-existing weight file, if available + + weight_file = None + if self.config.weightsDir is not None: + grid_key = forcings.productName + weight_file = os.path.join(self.config.weightsDir, "ESMF_weight_{}.nc4".format(grid_key)) + + # check if file exists: + if os.path.exists(weight_file): + # read the data + try: + self.log_status("Loading cached ESMF weight object for {} from {}".format( + forcings.productName, weight_file), root_only=True) + + begin = time.monotonic() + forcings.regridObj = ESMF.RegridFromFile(forcings.esmf_field_in, + forcings.esmf_field_out, + weight_file) + end = time.monotonic() + + self.log_status("Finished loading weight object with ESMF, took {} seconds".format(end - begin), + root_only=True) + + except (IOError, ValueError, ESMF.ESMPyException) as esmf_error: + self.log_critical("Unable to load cached ESMF weight file: {}".format(esmf_error)) + + if forcings.regridObj is None: + self.log_status("Creating weight object from ESMF", root_only=True) + + with self.parallel_block(): + try: + begin = time.monotonic() + forcings.regridObj = ESMF.Regrid(forcings.esmf_field_in, + forcings.esmf_field_out, + src_mask_values=np.array([0]), + regrid_method=ESMF.RegridMethod.BILINEAR, + unmapped_action=ESMF.UnmappedAction.IGNORE, + filename=weight_file) + end = time.monotonic() + + self.log_status("Finished generating weight object with ESMF, took {} seconds".format(end - begin), + root_only=True) + except (RuntimeError, ImportError, ESMF.ESMPyException) as esmf_error: + self.log_critical("Unable to regrid input data from ESMF: " + str(esmf_error)) + # etype, value, tb = sys.exc_info() + # traceback.print_exception(etype, value, tb) + # print(forcings.esmf_field_in) + # print(forcings.esmf_field_out) + # print(np.array([0])) + + with self.parallel_block(): + # Run the regridding object on this test dataset. Check the output grid for + # any 0 values. + try: + forcings.esmf_field_out = forcings.regridObj(forcings.esmf_field_in, + forcings.esmf_field_out) + except ValueError as ve: + self.log_critical("Unable to extract regridded data from ESMF regridded field: " + str(ve)) + # delete bad cached file if it exists + if weight_file is not None: + if os.path.exists(weight_file): + os.remove(weight_file) + + forcings.regridded_mask[:, :] = forcings.esmf_field_out.data[:, :] + + def need_regrid_object(self, datasource, forcing_idx: int, input_forcings): + """ + Function for checking to see if regridding weights need to be + calculated (or recalculated). + :param datasource: + :param forcing_idx: + :param input_forcings: + :return: + """ + + with self.parallel_block(): + # If the destination ESMF field hasn't been created, create it here. + if not input_forcings.esmf_field_out: + try: + input_forcings.esmf_field_out = ESMF.Field(self.geo_meta.esmf_grid, + name=input_forcings.productName + '_FORCING_REGRIDDED') + except ESMF.ESMPyException as esmf_error: + self.log_critical( + "Unable to create {} destination ESMF field object: {}".format(input_forcings.productName, + esmf_error)) + + # Determine if we need to calculate a regridding object. The following situations warrant the calculation of + # a new weight file: + # 1.) This is the first output time step, so we need to calculate a weight file. + # 2.) The input forcing grid has changed. + calc_regrid_flag = False + + if input_forcings.nx_global is None or input_forcings.ny_global is None: + # This is the first timestep. + # Create out regridded numpy arrays to hold the regridded data. + # TODO: could this be moved to where the Regrid is actually created? + input_forcings.regridded_forcings1 = np.empty([input_forcings.NUM_OUTPUTS, + self.geo_meta.ny_local, self.geo_meta.nx_local], + np.float32) + input_forcings.regridded_forcings2 = np.empty([input_forcings.NUM_OUTPUTS, + self.geo_meta.ny_local, self.geo_meta.nx_local], + np.float32) + + with self.parallel_block(): + if self.IS_MPI_ROOT: + if input_forcings.nx_global is None or input_forcings.ny_global is None: + # This is the first timestep. + calc_regrid_flag = True + else: + if datasource.variables[input_forcings.netcdf_var_names[forcing_idx]].shape[1] \ + != input_forcings.ny_global and \ + datasource.variables[input_forcings.netcdf_var_names[forcing_idx]].shape[2] \ + != input_forcings.nx_global: + calc_regrid_flag = True + + # Broadcast the flag to the other processors. + calc_regrid_flag = self.mpi.broadcast_parameter(calc_regrid_flag, self.config, param_type=bool) + + return calc_regrid_flag + + def already_processed(self, input_forcings): + # If the expected file is missing, this means we are allowing missing files, simply + # exit out of this routine as the regridded fields have already been set to NDV. + if not os.path.isfile(input_forcings.file_in2): + self.log_status("No {} input file found for this timestep".format(self.name), root_only=True) + return + + # Check to see if the regrid complete flag for this + # output time step is true. This entails the necessary + # inputs have already been regridded and we can move on. + if input_forcings.regridComplete: + self.log_status("No {} regridding required for this timestep".format(self.name), root_only=True) + return diff --git a/core/Regridding/regrid_cfs.py b/core/Regridding/regrid_cfs.py new file mode 100644 index 0000000..68132ea --- /dev/null +++ b/core/Regridding/regrid_cfs.py @@ -0,0 +1,46 @@ +import numpy as np + +from .regrid_grib2 import RegridGRIB2 + + +class RegridCFS(RegridGRIB2): + def __init__(self, config_options, wrf_hydro_geo_meta, mpi_config): + super().__init__("CFSv2", config_options, wrf_hydro_geo_meta, mpi_config) + + def regrid_input(self, input_forcings): + datasource = self.setup_input(input_forcings) + self.init_regrid_objects(datasource, input_forcings) + + # only run regrid if NOT using bias correction + if not self.config.runCfsNldasBiasCorrect: + self.regrid_fields(datasource, input_forcings) + input_forcings.coarse_input_forcings1 = input_forcings.coarse_input_forcings2 = None + else: + # bias correction will be run later: + for idx, nc_var in enumerate(input_forcings.netcdf_var_names): + var_local_tmp = self.extract_field(datasource, input_forcings, nc_var) + + imo_idx = input_forcings.input_map_output[idx] + input_forcings.regridded_forcings1[imo_idx, :, :] = self.config.globalNdv + input_forcings.regridded_forcings2[imo_idx, :, :] = self.config.globalNdv + if input_forcings.coarse_input_forcings1 is None: + input_forcings.coarse_input_forcings1 = np.empty([input_forcings.NUM_OUTPUTS, + var_local_tmp.shape[0], + var_local_tmp.shape[1]], + np.float64) + + if input_forcings.coarse_input_forcings2 is None: + input_forcings.coarse_input_forcings2 = np.empty_like(input_forcings.coarse_input_forcings1) + + try: + input_forcings.coarse_input_forcings2[imo_idx, :, :] = var_local_tmp + except (ValueError, KeyError, AttributeError) as err: + self.log_critical( + "Unable to place local {} input field {} into local numpy array:\n\t({})".format(self.name, + nc_var, err)) + + if self.config.current_output_step == 1: + input_forcings.coarse_input_forcings1[imo_idx, :, :] = \ + input_forcings.coarse_input_forcings2[imo_idx, :, :] + + self.release_netcdf(datasource) diff --git a/core/Regridding/regrid_grib2.py b/core/Regridding/regrid_grib2.py new file mode 100644 index 0000000..67aae88 --- /dev/null +++ b/core/Regridding/regrid_grib2.py @@ -0,0 +1,185 @@ +import os + +import numpy as np + +from .regrid_base import Regridder +from core import file_io + + +class RegridGRIB2(Regridder): + def __init__(self, regridder_name: str, config_options, wrf_hydro_geo_meta, mpi_config): + super().__init__(regridder_name, config_options, wrf_hydro_geo_meta, mpi_config) + + # TODO: Move this to regrid_base if possible + self._did_init_regrid = True + + def regrid_input(self, input_forcings): + datasource = self.setup_input(input_forcings) + self.init_regrid_objects(datasource, input_forcings) + self.regrid_fields(datasource, input_forcings) + self.release_netcdf(datasource) + + def setup_input(self, input_forcings): + if self.already_processed(input_forcings): + return + + # Create a path for a temporary NetCDF files that will + # be created through the wgrib2 process. + input_forcings.tmpFile = os.path.join(self.config.scratch_dir, "{}_TMP.nc".format(self.name, + self.mkfilename())) + + # This file shouldn't exist, but if it does (previously failed execution of the program), remove it... + if self.IS_MPI_ROOT: + if os.path.isfile(input_forcings.tmpFile): + self.log_status("Found old temporary file: {} - Removing...".format(input_forcings.tmpFile)) + try: + os.remove(input_forcings.tmpFile) + except OSError: + self.log_critical("Unable to remove temporary file: " + input_forcings.tmpFile) + + fields = [] + for forcing_idx, grib_var in enumerate(input_forcings.grib_vars): + self.log_status("Converting {} Variable: {}".format(self.name, grib_var), root_only=True) + fields.append(':' + grib_var + ':' + + input_forcings.grib_levels[forcing_idx] + ':' + + str(input_forcings.fcst_hour2) + " hour fcst:") + fields.append(":(HGT):(surface):") # add HGT variable to main set (no separate file) + + with self.parallel_block(): + # Create a temporary NetCDF file from the GRIB2 file. + cmd = '$WGRIB2 -match "(' + '|'.join(fields) + ')" ' + input_forcings.file_in2 + \ + " -netcdf " + input_forcings.tmpFile + + datasource = file_io.open_grib2(input_forcings.file_in2, input_forcings.tmpFile, cmd, + self.config, self.mpi, inputVar=None) + + return datasource + + def init_regrid_objects(self, datasource, input_forcings): + # TODO: why is there a loop here? Don't we only need one Regrid() object? + # TODO: move this to regrid_base if possible + for forcing_idx, grib_var in enumerate(input_forcings.grib_vars): + # self.log_status("Processing {} Variable: {}".format(self.name, grib_var), root_only=True) + + if self.need_regrid_object(datasource, forcing_idx, input_forcings): + with self.parallel_block(): + self.log_status("Calculating {} regridding weights".format(self.name), root_only=True) + self.calculate_weights(datasource, forcing_idx, input_forcings) + + # Regrid the height variable. + with self.parallel_block(): + var_tmp = None + if self.IS_MPI_ROOT: + try: + var_tmp = datasource.variables['HGT_surface'][0, :, :] + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to extract {} elevation from {}: {}".format( + self.name, input_forcings.tmpFile, err)) + + var_local_tmp = self.mpi.scatter_array(input_forcings, var_tmp, self.config) + + with self.parallel_block(): + try: + input_forcings.esmf_field_in.data[:, :] = var_local_tmp + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to place input {} data into ESMF field: {}".format(self.name, err)) + + with self.parallel_block(): + self.log_status("Regridding {} surface elevation data to the WRF-Hydro domain.".format(self.name), + root_only=True) + try: + input_forcings.esmf_field_out = input_forcings.regridObj(input_forcings.esmf_field_in, + input_forcings.esmf_field_out) + except ValueError as ve: + self.log_critical("Unable to regrid {} surface elevation using ESMF: {}".format(self.name, ve)) + + # Set any pixel cells outside the input domain to the global missing value. + with self.parallel_block(): + try: + input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ + self.config.globalNdv + except (ValueError, ArithmeticError) as npe: + self.log_critical( + "Unable to perform {} mask search on elevation data: {}".format(self.name, npe)) + + try: + input_forcings.height[:, :] = input_forcings.esmf_field_out.data + except (ValueError, KeyError, AttributeError) as err: + self.log_critical( + "Unable to extract regridded {} elevation data from ESMF: {}".format(self.name, err)) + + self._did_init_regrid = True + + def regrid_fields(self, datasource, input_forcings): + + if not self._did_init_regrid: + raise RuntimeError("Attempting to regrid {} without ESMF regridding objects initialized".format(self.name)) + + for forcing_idx, nc_var in enumerate(input_forcings.netcdf_var_names): + # EXTRACT AND REGRID THE INPUT FIELDS: + var_local_tmp = self.extract_field(datasource, input_forcings, nc_var) + + with self.parallel_block(): + try: + input_forcings.esmf_field_in.data[:, :] = var_local_tmp + except (ValueError, KeyError, AttributeError) as err: + self.log_critical("Unable to place input {} data into ESMF field: {}".format(self.name, err)) + + with self.parallel_block(): + self.log_status("Regridding {} input field: {}".format(self.name, nc_var), root_only=True) + try: + input_forcings.esmf_field_out = input_forcings.regridObj(input_forcings.esmf_field_in, + input_forcings.esmf_field_out) + except ValueError as ve: + self.log_critical("Unable to regrid {} input forcing data: {}".format(self.name, ve)) + + # Set any pixel cells outside the input domain to the global missing value. + with self.parallel_block(): + try: + input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ + self.config.globalNdv + except (ValueError, ArithmeticError) as npe: + self.log_critical("Unable to perform mask test on regridded {} forcings: {}".format(self.name, npe)) + + with self.parallel_block(): + try: + input_forcings.regridded_forcings2[input_forcings.input_map_output[forcing_idx], :, :] = \ + input_forcings.esmf_field_out.data + except (ValueError, KeyError, AttributeError) as err: + self.log_critical( + "Unable to extract regridded {} forcing data from ESMF field: {}".format(self.name, err)) + + # If we are on the first timestep, set the previous regridded field to be + # the latest as there are no states for time 0. + if self.config.current_output_step == 1: + input_forcings.regridded_forcings1[input_forcings.input_map_output[forcing_idx], :, :] = \ + input_forcings.regridded_forcings2[input_forcings.input_map_output[forcing_idx], :, :] + + def release_netcdf(self, datasource): + # Close the temporary NetCDF file and remove it. + if self.IS_MPI_ROOT: + ds_path = datasource.filepath() + try: + datasource.close() + except OSError: + self.log_critical("Unable to close NetCDF file: {}".format(ds_path)) + + try: + os.remove(ds_path) + except OSError: + # TODO: could this be just a warning? + self.log_critical("Unable to remove NetCDF file: {}".format(ds_path)) + + def extract_field(self, datasource, input_forcings, nc_var): + var_tmp = None + with self.parallel_block(): + if self.IS_MPI_ROOT: + self.log_status("Processing input {} variable: {}".format(self.name, nc_var)) + try: + var_tmp = datasource.variables[nc_var][0, :, :] + except (ValueError, KeyError, AttributeError) as err: + self.log_critical( + "Unable to extract {} from {} ({})".format(nc_var, input_forcings.tmpFile, err)) + + var_local_tmp = self.mpi.scatter_array(input_forcings, var_tmp, self.config) + return var_local_tmp diff --git a/core/Regridding/regrid_hrrr.py b/core/Regridding/regrid_hrrr.py new file mode 100644 index 0000000..a5e2713 --- /dev/null +++ b/core/Regridding/regrid_hrrr.py @@ -0,0 +1,6 @@ +from .regrid_grib2 import RegridGRIB2 + + +class RegridHRRR(RegridGRIB2): + def __init__(self, config_options, wrf_hydro_geo_meta, mpi_config): + super().__init__("CONUS HRRR", config_options, wrf_hydro_geo_meta, mpi_config) diff --git a/core/Regridding/regrid_netcdf.py b/core/Regridding/regrid_netcdf.py new file mode 100644 index 0000000..8201600 --- /dev/null +++ b/core/Regridding/regrid_netcdf.py @@ -0,0 +1,103 @@ +import os + +from core import file_io +from core.regridding.regrid_base import Regridder + + +class RegridCustomHourlyNetCDF(Regridder): + def __init__(self, config_options, wrf_hydro_geo_meta, mpi_config): + super().__init__("Custom Hourly NetCDF", config_options, wrf_hydro_geo_meta, mpi_config) + + # TODO: Move this to regrid_base if possible + self._did_init_regrid = True + + def regrid_input(self, input_forcings): + if self.already_processed(input_forcings): + return + + # Open the input NetCDF file containing necessary data. + datasource = file_io.open_netcdf_forcing(input_forcings.file_in2, self.config, self.mpi) + + for force_count, nc_var in enumerate(input_forcings.netcdf_var_names): + self.log_status("Processing Custom NetCDF Forcing Variable: ".format(nc_var)) + + calc_regrid_flag = check_regrid_status(id_tmp, force_count, input_forcings, + config_options, wrf_hydro_geo_meta, mpi_config) + + if calc_regrid_flag: + calculate_weights(id_tmp, force_count, input_forcings, config_options, mpi_config) + + # Read in the RAP height field, which is used for downscaling purposes. + if 'HGT_surface' not in id_tmp.variables.keys(): + config_options.errMsg = "Unable to locate HGT_surface in: " + input_forcings.file_in2 + raise Exception() + # mpi_config.comm.barrier() + + # Regrid the height variable. + if mpi_config.rank == 0: + var_tmp = id_tmp.variables['HGT_surface'][0, :, :] + else: + var_tmp = None + # mpi_config.comm.barrier() + + var_sub_tmp = mpi_config.scatter_array(input_forcings, var_tmp, config_options) + # mpi_config.comm.barrier() + + input_forcings.esmf_field_in.data[:, :] = var_sub_tmp + # mpi_config.comm.barrier() + + input_forcings.esmf_field_out = input_forcings.regridObj(input_forcings.esmf_field_in, + input_forcings.esmf_field_out) + # Set any pixel cells outside the input domain to the global missing value. + input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ + config_options.globalNdv + # mpi_config.comm.barrier() + + input_forcings.height[:, :] = input_forcings.esmf_field_out.data + # mpi_config.comm.barrier() + + # mpi_config.comm.barrier() + + # Regrid the input variables. + if mpi_config.rank == 0: + var_tmp = id_tmp.variables[input_forcings.netcdf_var_names[force_count]][0, :, :] + else: + var_tmp = None + # mpi_config.comm.barrier() + + var_sub_tmp = mpi_config.scatter_array(input_forcings, var_tmp, config_options) + # mpi_config.comm.barrier() + + input_forcings.esmf_field_in.data[:, :] = var_sub_tmp + # mpi_config.comm.barrier() + + input_forcings.esmf_field_out = input_forcings.regridObj(input_forcings.esmf_field_in, + input_forcings.esmf_field_out) + # Set any pixel cells outside the input domain to the global missing value. + input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ + config_options.globalNdv + # mpi_config.comm.barrier() + + input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.esmf_field_out.data + # mpi_config.comm.barrier() + + # If we are on the first timestep, set the previous regridded field to be + # the latest as there are no states for time 0. + if config_options.current_output_step == 1: + input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + # mpi_config.comm.barrier() + + # Close the temporary NetCDF file and remove it. + if mpi_config.rank == 0: + try: + id_tmp.close() + except OSError: + config_options.errMsg = "Unable to close NetCDF file: " + input_forcings.tmpFile + err_handler.err_out(config_options) + try: + os.remove(input_forcings.tmpFile) + except OSError: + config_options.errMsg = "Unable to remove NetCDF file: " + input_forcings.tmpFile + err_handler.err_out(config_options) diff --git a/core/Regridding/regrid_rap.py b/core/Regridding/regrid_rap.py new file mode 100644 index 0000000..4720062 --- /dev/null +++ b/core/Regridding/regrid_rap.py @@ -0,0 +1,6 @@ +from .regrid_grib2 import RegridGRIB2 + + +class RegridRAP(RegridGRIB2): + def __init__(self, config_options, wrf_hydro_geo_meta, mpi_config): + super().__init__("CONUS RAP", config_options, wrf_hydro_geo_meta, mpi_config) diff --git a/core/bias_correction.py b/core/bias_correction.py index c985811..7bf98f8 100755 --- a/core/bias_correction.py +++ b/core/bias_correction.py @@ -11,9 +11,9 @@ import numpy as np from netCDF4 import Dataset - +from strenum import StrEnum from core import err_handler - +from core.enumConfig import BiasCorrTempEnum, BiasCorrPressEnum, BiasCorrHumidEnum, BiasCorrWindEnum, BiasCorrSwEnum, BiasCorrLwEnum, BiasCorrPrecipEnum, DownScaleHumidEnum PARAM_NX = 384 PARAM_NY = 190 @@ -38,62 +38,64 @@ def run_bias_correction(input_forcings, config_options, geo_meta_wrf_hydro, mpi_ """ # Dictionary for mapping to temperature bias correction. bias_correct_temperature = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct, - 2: ncar_tbl_correction, - 3: ncar_temp_gfs_bias_correct, - 4: ncar_temp_hrrr_bias_correct + str(BiasCorrTempEnum.NONE.name) : no_bias_correct, + str(BiasCorrTempEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct, + str(BiasCorrTempEnum.CUSTOM.name): ncar_tbl_correction, + str(BiasCorrTempEnum.GFS.name) : ncar_temp_gfs_bias_correct, + str(BiasCorrTempEnum.HRRR.name) : ncar_temp_hrrr_bias_correct } + bias_correct_temperature[input_forcings.t2dBiasCorrectOpt](input_forcings, config_options, mpi_config, 0) + err_handler.check_program_status(config_options, mpi_config) # Dictionary for mapping to humidity bias correction. bias_correct_humidity = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct, - 2: ncar_tbl_correction + str(BiasCorrHumidEnum.NONE.name) : no_bias_correct, + str(BiasCorrHumidEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct, + str(BiasCorrHumidEnum.CUSTOM.name): ncar_tbl_correction } bias_correct_humidity[input_forcings.q2dBiasCorrectOpt](input_forcings, config_options, mpi_config, 1) err_handler.check_program_status(config_options, mpi_config) # Dictionary for mapping to surface pressure bias correction. bias_correct_pressure = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct + str(BiasCorrPressEnum.NONE.name) : no_bias_correct, + str(BiasCorrPressEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct } bias_correct_pressure[input_forcings.psfcBiasCorrectOpt](input_forcings, config_options, mpi_config, 7) err_handler.check_program_status(config_options, mpi_config) # Dictionary for mapping to incoming shortwave radiation correction. bias_correct_sw = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct, - 2: ncar_sw_hrrr_bias_correct + str(BiasCorrSwEnum.NONE.name) : no_bias_correct, + str(BiasCorrSwEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct, + str(BiasCorrSwEnum.CUSTOM.name): ncar_sw_hrrr_bias_correct } - if input_forcings.swBiasCorrectOpt != 2: + if input_forcings.swBiasCorrectOpt != str(BiasCorrSwEnum.CUSTOM.name): bias_correct_sw[input_forcings.swBiasCorrectOpt](input_forcings, config_options, mpi_config, 5) else: bias_correct_sw[input_forcings.swBiasCorrectOpt](input_forcings, geo_meta_wrf_hydro, - config_options, mpi_config, 5) + config_options, mpi_config, force_sw_num) err_handler.check_program_status(config_options, mpi_config) # Dictionary for mapping to incoming longwave radiation correction. bias_correct_lw = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct, - 2: ncar_blanket_adjustment_lw, - 3: ncar_lwdown_gfs_bias_correct + str(BiasCorrLwEnum.NONE.name) : no_bias_correct, + str(BiasCorrLwEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct, + str(BiasCorrLwEnum.CUSTOM.name): ncar_blanket_adjustment_lw, + str(BiasCorrLwEnum.GFS.name) : ncar_lwdown_gfs_bias_correct } bias_correct_lw[input_forcings.lwBiasCorrectOpt](input_forcings, config_options, mpi_config, 6) err_handler.check_program_status(config_options, mpi_config) # Dictionary for mapping to wind bias correction. bias_correct_wind = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct, - 2: ncar_tbl_correction, - 3: ncar_wspd_gfs_bias_correct, - 4: ncar_wspd_hrrr_bias_correct + str(BiasCorrWindEnum.NONE.name) : no_bias_correct, + str(BiasCorrWindEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct, + str(BiasCorrWindEnum.CUSTOM.name): ncar_tbl_correction, + str(BiasCorrWindEnum.GFS.name) : ncar_wspd_gfs_bias_correct, + str(BiasCorrWindEnum.HRRR.name) : ncar_wspd_hrrr_bias_correct } # Run for U-Wind bias_correct_wind[input_forcings.windBiasCorrectOpt](input_forcings, config_options, mpi_config, 2) @@ -104,15 +106,15 @@ def run_bias_correction(input_forcings, config_options, geo_meta_wrf_hydro, mpi_ # Dictionary for mapping to precipitation bias correction. bias_correct_precip = { - 0: no_bias_correct, - 1: cfsv2_nldas_nwm_bias_correct + str(BiasCorrPrecipEnum.NONE.name) : no_bias_correct, + str(BiasCorrPrecipEnum.CFS_V2.name): cfsv2_nldas_nwm_bias_correct } bias_correct_precip[input_forcings.precipBiasCorrectOpt](input_forcings, config_options, mpi_config, 4) err_handler.check_program_status(config_options, mpi_config) # Assign the temperature/pressure grids to temporary placeholders here. # these will be used if 2-meter specific humidity is downscaled. - if input_forcings.q2dDownscaleOpt != 0: + if input_forcings.q2dDownscaleOpt != str(DownScaleHumidEnum.NONE.name): input_forcings.t2dTmp = input_forcings.final_forcings[4, :, :] * 1.0 input_forcings.psfcTmp = input_forcings.final_forcings[6, :, :] * 1.0 else: @@ -176,8 +178,14 @@ def ncar_tbl_correction(input_forcings, config_options, mpi_config, force_num): # Extract local array of values to perform adjustment on. force_tmp = None + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + try: - force_tmp = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + force_tmp = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract: " + input_forcings.netcdf_var_names[force_num] + \ " from local forcing object for: " + input_forcings.productName + \ @@ -206,7 +214,7 @@ def ncar_tbl_correction(input_forcings, config_options, mpi_config, force_num): err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = force_tmp[:, :] + input_forcings.final_forcings[outId, :, :] = force_tmp[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary LW array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -263,10 +271,16 @@ def ncar_blanket_adjustment_lw(input_forcings, config_options, mpi_config, force diurnal_ampl_SR * math.sin(diurnal_offs_SR + hh / 24 * 2*math.pi) + \ monthly_ampl_SR * math.sin(monthly_offs_SR + MM / 12 * 2*math.pi) + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + # Perform adjustment. lw_tmp = None try: - lw_tmp = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + lw_tmp = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming LW from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -291,7 +305,7 @@ def ncar_blanket_adjustment_lw(input_forcings, config_options, mpi_config, force err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = lw_tmp[:, :] + input_forcings.final_forcings[outId, :, :] = lw_tmp[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary LW array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -370,7 +384,7 @@ def ncar_sw_hrrr_bias_correct(input_forcings, geo_meta_wrf_hydro, config_options # the cosine of the sol_zen_ang is proportional to the solar intensity # (not accounting for humidity or cloud cover) sol_cos = np.sin(geo_meta_wrf_hydro.latitude_grid * d2r) * math.sin(decl) + np.cos(geo_meta_wrf_hydro.latitude_grid * d2r) * math.cos(decl) * np.cos(ha) - + # Check for any values outside of [-1,1] (this can happen due to floating point rounding) sol_cos[np.where(sol_cos < -1.0)] = -1.0 sol_cos[np.where(sol_cos > 1.0)] = 1.0 @@ -384,9 +398,15 @@ def ncar_sw_hrrr_bias_correct(input_forcings, geo_meta_wrf_hydro, config_options # a local grid. We will perform the bias correction on this grid, based on forecast # hour and datetime information. Once a correction has taken place, we will place # the corrected field back into the forcing object. + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + sw_tmp = None try: - sw_tmp = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + sw_tmp = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming shortwave forcing from object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -476,9 +496,15 @@ def ncar_temp_hrrr_bias_correct(input_forcings, config_options, mpi_config, forc # config_options.statusMsg = f"\tAnAFlag = {config_options.ana_flag} {bias_corr}" # err_handler.log_msg(config_options, mpi_config) + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + temp_in = None try: - temp_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + temp_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming temperature from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -503,7 +529,7 @@ def ncar_temp_hrrr_bias_correct(input_forcings, config_options, mpi_config, forc err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = temp_in[:, :] + input_forcings.final_forcings[outId, :, :] = temp_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary temperature array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -532,9 +558,15 @@ def ncar_temp_gfs_bias_correct(input_forcings, config_options, mpi_config, force bias_corr = net_bias_mr + fhr_mult_mr * fhr + diurnal_ampl_mr * math.sin(diurnal_offs_mr + hh / 24 * TWO_PI) + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + temp_in = None try: - temp_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + temp_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming temperature from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -559,7 +591,7 @@ def ncar_temp_gfs_bias_correct(input_forcings, config_options, mpi_config, force err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = temp_in[:, :] + input_forcings.final_forcings[outId, :, :] = temp_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary temperature array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -590,9 +622,15 @@ def ncar_lwdown_gfs_bias_correct(input_forcings, config_options, mpi_config, for bias_corr = lwdown_net_bias_mr + lwdown_fhr_mult_mr * fhr + lwdown_diurnal_ampl_mr * \ math.sin(lwdown_diurnal_offs_mr + hh / 24 * TWO_PI) + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break + lwdown_in = None try: - lwdown_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + lwdown_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming longwave from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -617,7 +655,7 @@ def ncar_lwdown_gfs_bias_correct(input_forcings, config_options, mpi_config, for err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = lwdown_in[:, :] + input_forcings.final_forcings[outId, :, :] = lwdown_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary longwave array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -643,9 +681,17 @@ def ncar_wspd_hrrr_bias_correct(input_forcings, config_options, mpi_config, forc # need to get wind speed from U, V components ugrd_idx = input_forcings.grib_vars.index('UGRD') vgrd_idx = input_forcings.grib_vars.index('VGRD') + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[ugrd_idx]: + u_in = ind + elif xvar.name == input_forcings.input_map_output[vgrd_idx]: + v_in = ind + else: + continue - ugrid_in = input_forcings.final_forcings[input_forcings.input_map_output[ugrd_idx], :, :] - vgrid_in = input_forcings.final_forcings[input_forcings.input_map_output[vgrd_idx], :, :] + ugrid_in = input_forcings.final_forcings[u_in, :, :] + vgrid_in = input_forcings.final_forcings[v_in, :, :] wdir = np.arctan2(vgrid_in, ugrid_in) wspd = np.sqrt(np.square(ugrid_in) + np.square(vgrid_in)) @@ -685,10 +731,15 @@ def ncar_wspd_hrrr_bias_correct(input_forcings, config_options, mpi_config, forc # TODO: cache the "other" value so we don't repeat this calculation unnecessarily bias_corrected = ugrid_out if force_num == ugrd_idx else vgrid_out - + + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break wind_in = None try: - wind_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + wind_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming windspeed from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -713,7 +764,7 @@ def ncar_wspd_hrrr_bias_correct(input_forcings, config_options, mpi_config, forc err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = wind_in[:, :] + input_forcings.final_forcings[outId, :, :] = wind_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary windspeed array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -744,8 +795,17 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force ugrd_idx = input_forcings.grib_vars.index('UGRD') vgrd_idx = input_forcings.grib_vars.index('VGRD') - ugrid_in = input_forcings.final_forcings[input_forcings.input_map_output[ugrd_idx], :, :] - vgrid_in = input_forcings.final_forcings[input_forcings.input_map_output[vgrd_idx], :, :] + OutputEnum = config_options.OutputEnum + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[ugrd_idx]: + u_in = ind + elif xvar.name == input_forcings.input_map_output[vgrd_idx]: + v_in = ind + else: + continue + + ugrid_in = input_forcings.final_forcings[u_in, :, :] + vgrid_in = input_forcings.final_forcings[v_in, :, :] wdir = np.arctan2(vgrid_in, ugrid_in) wspd = np.sqrt(np.square(ugrid_in) + np.square(vgrid_in)) @@ -762,16 +822,19 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force # TODO: cache the "other" value so we don't repeat this calculation unnecessarily bias_corrected = ugrid_out if force_num == ugrd_idx else vgrid_out - + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break wind_in = None try: - wind_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] + wind_in = input_forcings.final_forcings[outId, :, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to extract incoming windspeed from forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) - + ind_valid = None try: ind_valid = np.where(wind_in != config_options.globalNdv) @@ -790,7 +853,7 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = wind_in[:, :] + input_forcings.final_forcings[outId, :, :] = wind_in[:, :] except NumpyExceptions as npe: config_options.errMsg = "Unable to place temporary windspeed array back into forcing object for: " + \ input_forcings.productName + " (" + str(npe) + ")" @@ -908,6 +971,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for # Open the NetCDF file. try: id_nldas_param = Dataset(nldas_param_file, 'r') + id_nldas_param.set_auto_mask(False) # don't mask missing since it won't survive broadcast except OSError as err: config_options.errMsg = "Unable to open parameter file: " + nldas_param_file + " (" + str(err) + ")" err_handler.log_critical(config_options, mpi_config) @@ -1016,6 +1080,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for nldas_param_1 = None nldas_param_2 = None nldas_zero_pcp = None + err_handler.check_program_status(config_options, mpi_config) # Scatter NLDAS parameters @@ -1236,7 +1301,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for # regridding object. # 6.) Place the data into the final output arrays for further processing (downscaling). # 7.) Reset variables for memory efficiency and exit the routine. - + np.seterr(all='ignore') if mpi_config.rank == 0: @@ -1252,12 +1317,15 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for config_options.statusMsg = "Looping over local arrays to calculate bias corrections." err_handler.log_msg(config_options, mpi_config) # Process each of the pixel cells for this local processor on the CFS grid. + outId = None + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_num]: + outId = ind + break for x_local in range(0, input_forcings.nx_local): for y_local in range(0, input_forcings.ny_local): - cfs_prev_tmp = input_forcings.coarse_input_forcings1[input_forcings.input_map_output[force_num], - y_local, x_local] - cfs_next_tmp = input_forcings.coarse_input_forcings2[input_forcings.input_map_output[force_num], - y_local, x_local] + cfs_prev_tmp = input_forcings.coarse_input_forcings1[outId, y_local, x_local] + cfs_next_tmp = input_forcings.coarse_input_forcings2[outId, y_local, x_local] # Check for any missing parameter values. If any missing values exist, # set this flag to False. Further down, if it's False, we will simply @@ -1479,9 +1547,9 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = \ + input_forcings.final_forcings[outId, :, :] = \ input_forcings.esmf_field_out.data except NumpyExceptions as npe: config_options.errMsg = "Unable to extract ESMF field data for CFSv2: " + str(npe) err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) \ No newline at end of file diff --git a/core/config.py b/core/config.py index bc70e22..ac022a5 100755 --- a/core/config.py +++ b/core/config.py @@ -1,10 +1,15 @@ -import configparser +#!/usr/bin/env python + import datetime import json import os - +import sys +from enum import IntEnum +import enum +import configparser import numpy as np - +import yaml +from strenum import StrEnum from core import time_handling from core import err_handler @@ -20,10 +25,10 @@ def __init__(self, config): Initialize the configuration class to empty None attributes param config: The user-specified path to the configuration file. """ - self.input_forcings = None + #self.input_forcings = None self.supp_precip_forcings = None - self.input_force_dirs = None - self.input_force_types = None + #self.input_force_dirs = None + #self.input_force_types = None self.supp_precip_dirs = None self.supp_precip_file_types = None self.supp_precip_param_dir = None @@ -62,6 +67,7 @@ def __init__(self, config): self.process_window = None self.geogrid = None self.spatial_meta = None + self.grid_meta = None self.ignored_border_widths = None self.regrid_opt = None self.weightsDir = None @@ -92,12 +98,18 @@ def __init__(self, config): self.customFcstFreq = None self.rqiMethod = None self.rqiThresh = 1.0 - self.globalNdv = -9999.0 + self.globalNdv = -999999.0 self.d_program_init = datetime.datetime.utcnow() self.errFlag = 0 self.nwmVersion = None self.nwmConfig = None self.include_lqfraq = False + self.forcingInputModYaml = None + self.suppPrecipModYaml = None + self.outputVarAttrYaml = None + self.ForcingEnum = None + self.SuppForcingPcpEnum = None + self.OutputEnum = None def read_config(self): """ @@ -105,65 +117,100 @@ def read_config(self): were provided. """ # Read in the configuration file - config = configparser.ConfigParser() + yaml_stream = None + try: + yaml_stream = open(self.config_path) + config = yaml.safe_load(yaml_stream) + except yaml.YAMLError as yaml_exc: + err_handler.err_out_screen('Error parsing the configuration file: %s\n%s' % (self.config_path,yaml_exc)) + except IOError: + err_handler.err_out_screen('Unable to open the configuration file: %s' % self.config_path) + finally: + if yaml_stream: + yaml_stream.close() + try: + inputs = config['Input'] + except KeyError: + raise KeyError("Unable to locate Input map in configuration file.") + err_handler.err_out_screen('Unable to locate Input map in configuration file.') + + if len(inputs) == 0: + err_handler.err_out_screen('Please choose at least one Forcings dataset to process') + + self.number_inputs = len(inputs) + + # Create Enums dynamically from the yaml files + try: + yaml_config = config['YamlConfig'] + except KeyError: + raise KeyError("Unable to locate YamlConfig in configuration file.") + err_handler.err_out_screen('Unable to locate YamlConfig in configuration file.') + + try: + self.forcingInputModYaml = yaml_config['forcingInputModYaml'] + except KeyError: + raise KeyError("Unable to locate forcingInputModYaml in configuration file.") + err_handler.err_out_screen('Unable to locate forcingInputModYaml in configuration file.') + + forcing_yaml_stream = open(self.forcingInputModYaml) + forcingConfig = yaml.safe_load(forcing_yaml_stream) + dynamicForcing = {} + for k in forcingConfig.keys(): + dynamicForcing[k] = k + self.ForcingEnum = enum.Enum('ForcingEnum', dynamicForcing) + + try: + self.suppPrecipModYaml = yaml_config['suppPrecipModYaml'] + except KeyError: + raise KeyError("Unable to locate suppPrecipModYaml in configuration file.") + err_handler.err_out_screen('Unable to locate suppPrecipModYaml in configuration file.') + + supp_yaml_stream = open(self.suppPrecipModYaml) + suppConfig = yaml.safe_load(supp_yaml_stream) + dynamicSupp = {} + for k in suppConfig.keys(): + dynamicSupp[k] = k + self.SuppForcingPcpEnum = enum.Enum('SuppForcingPcpEnum', dynamicSupp) try: - config.read(self.config_path) + self.outputVarAttrYaml = yaml_config['outputVarAttrYaml'] except KeyError: - err_handler.err_out_screen('Unable to open the configuration file: ' + self.config_path) + raise KeyError("Unable to locate outputVarAttrYaml in configuration file.") + err_handler.err_out_screen('Unable to locate outputVarAttrYaml in configuration file.') + + out_yaml_stream = open(self.outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) + dynamicOut = {} + for k in outConfig.keys(): + dynamicOut[k] = k + self.OutputEnum = enum.Enum('OutputEnum', dynamicOut) # Read in the base input forcing options as an array of values to map. try: - self.input_forcings = json.loads(config['Input']['InputForcings']) + self.input_forcings = [input['Forcing'] for input in inputs] except KeyError: - err_handler.err_out_screen('Unable to locate InputForcings under Input section in configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate InputForcings under Input section in configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper InputForcings option specified in configuration file') - if len(self.input_forcings) == 0: - err_handler.err_out_screen('Please choose at least one InputForcings dataset to process') - self.number_inputs = len(self.input_forcings) - - # Check to make sure forcing options make sense + err_handler.err_out_screen('Please pick Input[i][\'Forcing\'] from options: %s' % [str(item) for item in self.ForcingEnum]) + for forceOpt in self.input_forcings: - if forceOpt < 0 or forceOpt > 20: - err_handler.err_out_screen('Please specify InputForcings values between 1 and 20.') # Keep tabs on how many custom input forcings we have. - if forceOpt == 10: + if "CUSTOM" in forceOpt: self.number_custom_inputs = self.number_custom_inputs + 1 # Read in the input forcings types (GRIB[1|2], NETCDF) try: - self.input_force_types = config.get('Input', 'InputForcingTypes').strip("[]").split(',') - self.input_force_types = [ftype.strip() for ftype in self.input_force_types] - if self.input_force_types == ['']: - self.input_force_types = [] + self.input_force_types = [input['Type'] for input in inputs] except KeyError: - err_handler.err_out_screen('Unable to locate InputForcingTypes in Input section ' - 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate InputForcingTypes in Input section ' - 'in the configuration file.') + err_handler.err_out_screen('Please pick Input[i][\'Type\'] from options: %s' % [str(item) for item in ForcingTypeEnum]) if len(self.input_force_types) != self.number_inputs: - err_handler.err_out_screen('Number of InputForcingTypes must match the number ' - 'of InputForcings in the configuration file.') - for fileType in self.input_force_types: - if fileType not in ['GRIB1', 'GRIB2', 'NETCDF']: - err_handler.err_out_screen('Invalid forcing file type "{}" specified. ' - 'Only GRIB1, GRIB2, and NETCDF are supported'.format(fileType)) - + err_handler.err_out_screen('Number of Forcing Types must match the number ' + 'of Forcings in the configuration file.') # Read in the input directories for each forcing option. try: - self.input_force_dirs = config.get('Input', 'InputForcingDirectories').split(',') + self.input_force_dirs = [input['Dir'] for input in inputs] except KeyError: - err_handler.err_out_screen('Unable to locate InputForcingDirectories in Input section ' - 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate InputForcingDirectories in Input section ' - 'in the configuration file.') + err_handler.err_out_screen('Unable to locate Input[i][\'Dir\'] in Input map in the configuration file.') if len(self.input_force_dirs) != self.number_inputs: - err_handler.err_out_screen('Number of InputForcingDirectories must match the number ' - 'of InputForcings in the configuration file.') + err_handler.err_out_screen('Number of Input Directories must match the number ' + 'of Forcings in the configuration file.') # Loop through and ensure all input directories exist. Also strip out any whitespace # or new line characters. for dirTmp in range(0, len(self.input_force_dirs)): @@ -174,25 +221,12 @@ def read_config(self): # Read in the mandatory enforcement options for input forcings. try: - self.input_force_mandatory = json.loads(config['Input']['InputMandatory']) + self.input_force_mandatory = [int(input['Mandatory']) for input in inputs] except KeyError: - err_handler.err_out_screen('Unable to locate InputMandatory under Input section in configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate InputMandatory under Input section in configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper InputMandatory option specified in configuration file') + err_handler.err_out_screen('Missing Input[i][\'Mandatory\'] in Input map in configuration file.') + except ValueError: + err_handler.err_out_screen('Invalid Input[i][\'Mandatory\'] value in Input map in configuration file.') - # Process input forcing enforcement options - try: - self.input_force_mandatory = json.loads(config['Input']['InputMandatory']) - except KeyError: - err_handler.err_out_screen('Unable to locate InputMandatory under the Input section ' - 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate InputMandatory under the Input section ' - 'in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper InputMandatory options specified in the configuration file.') if len(self.input_force_mandatory) != self.number_inputs: err_handler.err_out_screen('Please specify InputMandatory values for each corresponding input ' 'forcings in the configuration file.') @@ -202,49 +236,346 @@ def read_config(self): err_handler.err_out_screen('Invalid InputMandatory chosen in the configuration file. Please' ' choose a value of 0 or 1 for each corresponding input forcing.') + # Read in the ForecastInputHorizons options. + try: + self.fcst_input_horizons = [input['Horizon'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastInputHorizons under Forecast section in ' + 'configuration file.') + if len(self.fcst_input_horizons) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[\'Horizon\'] values for ' + 'each corresponding input forcings for forecasts.') + # Check to make sure the horizons options make sense. There will be additional + # checking later when input choices are mapped to input products. + for horizonOpt in self.fcst_input_horizons: + if horizonOpt <= 0: + err_handler.err_out_screen('Please specify ForecastInputHorizon values greater ' + 'than zero.') + + # Read in the ForecastInputOffsets options. + try: + self.fcst_input_offsets = [input['Offset'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastInputOffsets under Forecast ' + 'section in the configuration file.') + if len(self.fcst_input_offsets) != self.number_inputs: + err_handler.err_out_screen('Please specify ForecastInputOffset values for each ' + 'corresponding input forcings for forecasts.') + # Check to make sure the input offset options make sense. There will be additional + # checking later when input choices are mapped to input products. + for inputOffset in self.fcst_input_offsets: + if inputOffset < 0: + err_handler.err_out_screen( + 'Please specify ForecastInputOffset values greater than or equal to zero.') + + # Check for the IgnoredBorderWidths + try: + self.ignored_border_widths = [input['IgnoredBorderWidths'] for input in inputs] + except KeyError: + # if didn't specify, no worries, just set to 0 + self.ignored_border_widths = [0.0]*self.number_inputs + if len(self.ignored_border_widths) != self.number_inputs: + err_handler.err_out_screen('Please specify IgnoredBorderWidths values for each ' + 'corresponding input forcings for SuppForcing.' + '({} was supplied'.format(self.ignored_border_widths)) + if any(map(lambda x: x < 0, self.ignored_border_widths)): + err_handler.err_out_screen('Please specify IgnoredBorderWidths values greater than or equal to zero:' + '({} was supplied'.format(self.ignored_border_widths)) + # Process regridding options. + try: + self.regrid_opt = [input['RegriddingOpt'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'RegriddingOpt\'] from options: %s' % [str(item) for item in RegriddingOptEnum]) + if len(self.regrid_opt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'RegriddingOpt\'] values for each corresponding input ' + 'forcings in the configuration file.') + + # Read in temporal interpolation options. + try: + self.forceTemoralInterp = [input['TemporalInterp'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'TemporalInterp\'] from options: %s' % [str(item) for item in TemporalInterpEnum]) + if len(self.forceTemoralInterp) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'TemporalInterp\'] values for each corresponding input forcings in the configuration file.') + + # Read in information for the custom input NetCDF files that are to be processed. + # Read in the ForecastInputHorizons options. + self.customFcstFreq = [input['Custom']['input_fcst_freq'] for input in inputs if 'Custom' in input and 'input_fcst_freq' in input['Custom']] + if len(self.customFcstFreq) != self.number_custom_inputs: + err_handler.err_out_screen('Improper custom_input fcst_freq specified. This number must ' + 'match the frequency of custom input forcings selected.') + + + # * Bias Correction Options * + try: + forecast = config['Forecast'] + except KeyError: + raise KeyError("Forecast not found in configuration file") + err_handler.err_out_screen('Unable to locate Forecast map in configuration file.') + + # Read AnA flag option + try: + self.ana_flag = int(forecast['AnAFlag']) + except KeyError: + err_handler.err_out_screen('Unable to locate Forecast[\'AnAFlag\'] in the configuration file.') + except ValueError: + err_handler.err_out_screen('Improper Forecast[\'AnAFlag\'] value ') + if self.ana_flag < 0 or self.ana_flag > 1: + err_handler.err_out_screen('Please choose a Forecast[\'AnAFlag\'] value of 0 or 1.') + + # Read in temperature bias correction options + try: + self.t2BiasCorrectOpt = [input['BiasCorrection']['Temperature'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Temperature\'] from options: %s' % [str(item) for item in BiasCorrTempEnum]) + if len(self.t2BiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Temperature\'] values for each corresponding ' + 'input forcings in the configuration file.') + + # Read in surface pressure bias correction options. + try: + self.psfcBiasCorrectOpt = [input['BiasCorrection']['Pressure'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Pressure\'] from options: %s' % [str(item) for item in BiasCorrPressEnum]) + if len(self.psfcBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Pressure\'] values for each corresponding ' + 'input forcings in the configuration file.') + + # Ensure the bias correction options chosen make sense. + for optTmp in self.psfcBiasCorrectOpt: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in humidity bias correction options. + try: + self.q2BiasCorrectOpt = [input['BiasCorrection']['Humidity'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Humidity\'] from options: %s' % [str(item) for item in BiasCorrHumidEnum]) + if len(self.q2BiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Humidity\'] values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.q2BiasCorrectOpt: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in wind bias correction options. + try: + self.windBiasCorrect = [input['BiasCorrection']['Wind'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Wind\'] from options: %s' % [str(item) for item in BiasCorrWindEnum]) + if len(self.windBiasCorrect) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Wind\'] values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.windBiasCorrect: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in shortwave radiation bias correction options. + try: + self.swBiasCorrectOpt = [input['BiasCorrection']['Shortwave'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Shortwave\'] from options: %s' % [str(item) for item in BiasCorrSwEnum]) + if len(self.swBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Shortwave\'] values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.swBiasCorrectOpt: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in longwave radiation bias correction options. + try: + self.lwBiasCorrectOpt = [input['BiasCorrection']['Longwave'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Longwave\'] from options: %s' % [str(item) for item in BiasCorrLwEnum]) + if len(self.lwBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Longwave\'] values for each corresponding ' + 'input forcings in the configuration file.') + + # Ensure the bias correction options chosen make sense. + for optTmp in self.lwBiasCorrectOpt: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in precipitation bias correction options. + try: + self.precipBiasCorrectOpt = [input['BiasCorrection']['Precip'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'BiasCorrection\'][\'Precip\'] from options: %s' % [str(item) for item in BiasCorrPrecipEnum]) + if len(self.precipBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'BiasCorrection\'][\'Precip\'] values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.precipBiasCorrectOpt: + if optTmp == 'CFS_V2': + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on + # for ALL variables. + if self.runCfsNldasBiasCorrect: + if self.precipBiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'Precipitation under this configuration.') + if self.lwBiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'long-wave radiation under this configuration.') + if self.swBiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'short-wave radiation under this configuration.') + if self.t2BiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'surface temperature under this configuration.') + if self.windBiasCorrect != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'wind forcings under this configuration.') + if self.q2BiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'specific humidity under this configuration.') + if self.psfcBiasCorrectOpt != ['CFS_V2']: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'surface pressure under this configuration.') + # Make sure we don't have any other forcings activated. This can only be ran for CFSv2. + for optTmp in self.input_forcings: + if optTmp != 'CFS_V2': + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction can only be used in ' + 'CFSv2-only configurations') + + # Read in the temperature downscaling options. + # Create temporary array to hold flags of if we need input parameter files. + param_flag = np.empty([len(self.input_forcings)], int) + param_flag[:] = 0 + try: + self.t2dDownscaleOpt = [input['Downscaling']['Temperature'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'Downscaling\'][\'Temperature\'] from options: %s' % [str(item) for item in DownScaleTempEnum]) + if len(self.t2dDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'Downscaling\'][\'Temperature\'] value for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + count_tmp = 0 + for optTmp in self.t2dDownscaleOpt: + if optTmp == 'LAPSE_PRE_CALC': + param_flag[count_tmp] = 1 + count_tmp = count_tmp + 1 + + # Read in the pressure downscaling options. + try: + self.psfcDownscaleOpt = [input['Downscaling']['Pressure'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'Downscaling\'][\'Pressure\'] from options: %s' % [str(item) for item in DownScalePressEnum]) + if len(self.psfcDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'Downscaling\'][\'Pressure\'] value for each corresponding ' + 'input forcings in the configuration file.') + + # Read in the shortwave downscaling options + try: + self.swDownscaleOpt = [input['Downscaling']['Shortwave'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'Downscaling\'][\'Shortwave\'] from options: %s' % [str(item) for item in DownScaleSwEnum]) + if len(self.swDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'Downscaling\'][\'Shortwave\'] value for each corresponding ' + 'input forcings in the configuration file.') + + # Read in the precipitation downscaling options + try: + self.precipDownscaleOpt = [input['Downscaling']['Precip'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'Downscaling\'][\'Precip\'] from options: %s' % [str(item) for item in DownScalePrecipEnum]) + if len(self.precipDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'Downscaling\'][\'Precip\'] value for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + count_tmp = 0 + for optTmp in self.precipDownscaleOpt: + if optTmp == 'NWM_MM': + param_flag[count_tmp] = 1 + count_tmp = count_tmp + 1 + + # Read in humidity downscaling options. + try: + self.q2dDownscaleOpt = [input['Downscaling']['Humidity'] for input in inputs] + except KeyError: + err_handler.err_out_screen('Please pick Input[i][\'Downscaling\'][\'Humidity\'] from options: %s' % [str(item) for item in DownScaleHumidEnum]) + if len(self.q2dDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify Input[i][\'Downscaling\'][\'Humidity\'] value for each corresponding ' + 'input forcings in the configuration file.') + + # Read in the downscaling parameter directory. + self.paramFlagArray = param_flag + tmp_scale_param_dirs = [] + if param_flag.sum() > 0: + tmp_scale_param_dirs = [input['Downscaling']['ParamDir'] for input in inputs if 'ParamDir' in input['Downscaling']] + if len(tmp_scale_param_dirs) < param_flag.sum(): + err_handler.err_out_screen('Please specify a Input[i][\'Downscaling\'][\'ParamDir\'] for each ' + 'corresponding downscaling option that requires one.') + # Loop through each downscaling parameter directory and make sure they exist. + for dirTmp in range(0, len(tmp_scale_param_dirs)): + tmp_scale_param_dirs[dirTmp] = tmp_scale_param_dirs[dirTmp].strip() + if not os.path.isdir(tmp_scale_param_dirs[dirTmp]): + err_handler.err_out_screen('Unable to locate parameter directory: ' + tmp_scale_param_dirs[dirTmp]) + + # Create a list of downscaling parameter directories for each corresponding + # input forcing. If no directory is needed, or specified, we will set the value to NONE + self.dScaleParamDirs = [] + for count_tmp, _ in enumerate(self.input_forcings): + if param_flag[count_tmp] == 0: + self.dScaleParamDirs.append('NONE') + if param_flag[count_tmp] == 1: + self.dScaleParamDirs.append(tmp_scale_param_dirs[count_tmp]) + + # if the directory was specified but not downscaling, set it anyway for bias correction etc. + if param_flag.sum() == 0 and len([input['Downscaling']['ParamDir'] for input in inputs if 'ParamDir' in input['Downscaling']]) >= 1: + self.dScaleParamDirs = [input['Downscaling']['ParamDir'] for input in inputs if 'ParamDir' in input['Downscaling']] + + try: + output = config['Output'] + except KeyError: + raise KeyError("Output not found in configuration file") + err_handler.err_out_screen('Unable to locate Output map in configuration file.') + # Read in the output frequency try: - self.output_freq = int(config['Output']['OutputFrequency']) + self.output_freq = int(output['Frequency']) except ValueError: err_handler.err_out_screen('Improper OutputFrequency value specified in the configuration file.') except KeyError: err_handler.err_out_screen('Unable to locate OutputFrequency in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate OutputFrequency in the configuration file.') if self.output_freq <= 0: err_handler.err_out_screen('Please specify an OutputFrequency that is greater than zero minutes.') # Read in the output directory try: - self.output_dir = config['Output']['OutDir'] + self.output_dir = output['Dir'] except ValueError: err_handler.err_out_screen('Improper OutDir specified in the configuration file.') except KeyError: err_handler.err_out_screen('Unable to locate OutDir in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate OutDir in the configuration file.') if not os.path.isdir(self.output_dir): err_handler.err_out_screen('Specified output directory: ' + self.output_dir + ' not found.') # Read in the scratch temporary directory. try: - self.scratch_dir = config['Output']['ScratchDir'] + self.scratch_dir = output['ScratchDir'] except ValueError: err_handler.err_out_screen('Improper ScratchDir specified in the configuration file.') except KeyError: err_handler.err_out_screen('Unable to locate ScratchDir in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ScratchDir in the configuration file.') if not os.path.isdir(self.scratch_dir): err_handler.err_out_screen('Specified output directory: ' + self.scratch_dir + ' not found') # Read in compression option try: - self.useCompression = int(config['Output']['compressOutput']) + self.useCompression = int(output['CompressOutput']) except KeyError: err_handler.err_out_screen('Unable to locate compressOut in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate compressOut in the configuration file.') except ValueError: err_handler.err_out_screen('Improper compressOut value.') if self.useCompression < 0 or self.useCompression > 1: @@ -252,17 +583,18 @@ def read_config(self): # Read in floating-point option try: - self.useFloats = int(config['Output']['floatOutput']) + self.useFloats = output['FloatOutput'] except KeyError: - # err_handler.err_out_screen('Unable to locate floatOutput in the configuration file.') - self.useFloats = 0 - except configparser.NoOptionError: - # err_handler.err_out_screen('Unable to locate floatOutput in the configuration file.') + # err_handler.err_out_screen('Unable to locate Output[\'FloatOutput\'] in the configuration file.') self.useFloats = 0 + err_handler.err_out_screen('Please pick output[\'FloatOutput\'] from options: %s' % [str(item) for item in OutputFloatEnum]) except ValueError: err_handler.err_out_screen('Improper floatOutput value: {}'.format(config['Output']['floatOutput'])) - if self.useFloats < 0 or self.useFloats > 1: - err_handler.err_out_screen('Please choose a floatOutput value of 0 or 1.') + + try: + retrospective = config['Retrospective'] + except KeyError: + err_handler.err_out_screen('Unable to locate Retrospective map in configuration file.') # Read in lqfrac option try: @@ -276,11 +608,11 @@ def read_config(self): except ValueError: err_handler.err_out_screen('Improper includeLQFraq value: {}'.format(config['Output']['includeLQFraq'])) if self.include_lqfraq < 0 or self.include_lqfraq > 1: - err_handler.err_out_screen('Please choose a includeLQFraq value of 0 or 1.') + err_handler.err_out_screen('Please choose an includeLQFraq value of 0 or 1.') # Read in retrospective options try: - self.retro_flag = int(config['Retrospective']['RetroFlag']) + self.retro_flag = retrospective['Flag'] except KeyError: err_handler.err_out_screen('Unable to locate RetroFlag in the configuration file.') except configparser.NoOptionError: @@ -295,7 +627,7 @@ def read_config(self): self.realtime_flag = False self.refcst_flag = False try: - beg_date_tmp = config['Retrospective']['BDateProc'] + beg_date_tmp = str(restrospective['BDateProc']) except KeyError: err_handler.err_out_screen('Unable to locate BDateProc under Logistics section in ' 'configuration file.') @@ -318,7 +650,7 @@ def read_config(self): # Process the ending date of retrospective forcings to process try: - end_date_tmp = config['Retrospective']['EDateProc'] + end_date_tmp = str(retrospective['EDateProc']) except KeyError: err_handler.err_out_screen('Unable to locate EDateProc under Logistics section in ' 'configuration file.') @@ -348,36 +680,34 @@ def read_config(self): # Calculate the number of output time steps dt_tmp = self.e_date_proc - self.b_date_proc self.num_output_steps = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.output_freq) - self.actual_output_steps = self.num_output_steps + if self.ana_flag: + self.actual_output_steps = np.int32(self.nFcsts) + else: + self.actual_output_steps = np.int32(self.num_output_steps) # Process realtime or reforecasting options. if self.retro_flag == 0: # If the retro flag is off, we are assuming a realtime or reforecast simulation. try: - self.look_back = int(config['Forecast']['LookBack']) + self.look_back = int(forecast['LookBack']) if self.look_back <= 0 and self.look_back != -9999: err_handler.err_out_screen('Please specify a positive LookBack or -9999 for realtime.') except ValueError: + raise ValueError("Improper value") err_handler.err_out_screen('Improper LookBack value entered into the ' 'configuration file. Please check your entry.') + except KeyError: err_handler.err_out_screen('Unable to locate LookBack in the configuration ' 'file. Please verify entries exist.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate LookBack in the configuration ' - 'file. Please verify entries exist.') # Process the beginning date of reforecast forcings to process try: - beg_date_tmp = config['Forecast']['RefcstBDateProc'] + beg_date_tmp = str(forecast['RefcstBDateProc']) except KeyError: err_handler.err_out_screen('Unable to locate RefcstBDateProc under Logistics section in ' 'configuration file.') beg_date_tmp = None - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RefcstBDateProc under Logistics section in ' - 'configuration file.') - beg_date_tmp = None if beg_date_tmp != '-9999': if len(beg_date_tmp) != 12: err_handler.err_out_screen('Improper RefcstBDateProc length entered into the ' @@ -385,6 +715,7 @@ def read_config(self): try: self.b_date_proc = datetime.datetime.strptime(beg_date_tmp, '%Y%m%d%H%M') except ValueError: + raise ValueError("Improper value") err_handler.err_out_screen('Improper RefcstBDateProc value entered into the ' 'configuration file. Please check your entry.') else: @@ -392,15 +723,11 @@ def read_config(self): # Process the ending date of reforecast forcings to process try: - end_date_tmp = config['Forecast']['RefcstEDateProc'] + end_date_tmp = str(forecast['RefcstEDateProc']) except KeyError: err_handler.err_out_screen('Unable to locate RefcstEDateProc under Logistics section in ' 'configuration file.') end_date_tmp = None - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RefcstEDateProc under Logistics section in ' - 'configuration file.') - end_date_tmp = None if end_date_tmp != '-9999': if len(end_date_tmp) != 12: err_handler.err_out_screen('Improper RefcstEDateProc length entered into the' @@ -440,16 +767,13 @@ def read_config(self): # Read in the ForecastFrequency option. try: - self.fcst_freq = int(config['Forecast']['ForecastFrequency']) + self.fcst_freq = int(forecast['Frequency']) except ValueError: err_handler.err_out_screen('Improper ForecastFrequency value entered into ' 'the configuration file. Please check your entry.') except KeyError: err_handler.err_out_screen('Unable to locate ForecastFrequency in the configuration ' 'file. Please verify entries exist.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ForecastFrequency in the configuration ' - 'file. Please verify entries exist.') if self.fcst_freq <= 0: err_handler.err_out_screen('Please specify a ForecastFrequency in the configuration ' 'file greater than zero.') @@ -463,16 +787,13 @@ def read_config(self): # it's used to calculate the beginning of the processing window. if True: # was: self.realtime_flag: try: - self.fcst_shift = int(config['Forecast']['ForecastShift']) + self.fcst_shift = int(config['Forecast']['Shift']) except ValueError: err_handler.err_out_screen('Improper ForecastShift value entered into the ' 'configuration file. Please check your entry.') except KeyError: err_handler.err_out_screen('Unable to locate ForecastShift in the configuration ' 'file. Please verify entries exist.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ForecastShift in the configuration ' - 'file. Please verify entries exist.') if self.fcst_shift < 0: err_handler.err_out_screen('Please specify a ForecastShift in the configuration ' 'file greater than or equal to zero.') @@ -493,51 +814,6 @@ def read_config(self): if self.look_back != -9999: time_handling.calculate_lookback_window(self) - # Read in the ForecastInputHorizons options. - try: - self.fcst_input_horizons = json.loads(config['Forecast']['ForecastInputHorizons']) - except KeyError: - err_handler.err_out_screen('Unable to locate ForecastInputHorizons under Forecast section in ' - 'configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ForecastInputHorizons under Forecast section in ' - 'configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper ForecastInputHorizons option specified in ' - 'configuration file') - if len(self.fcst_input_horizons) != self.number_inputs: - err_handler.err_out_screen('Please specify ForecastInputHorizon values for ' - 'each corresponding input forcings for forecasts.') - - # Check to make sure the horizons options make sense. There will be additional - # checking later when input choices are mapped to input products. - for horizonOpt in self.fcst_input_horizons: - if horizonOpt <= 0: - err_handler.err_out_screen('Please specify ForecastInputHorizon values greater ' - 'than zero.') - - # Read in the ForecastInputOffsets options. - try: - self.fcst_input_offsets = json.loads(config['Forecast']['ForecastInputOffsets']) - except KeyError: - err_handler.err_out_screen('Unable to locate ForecastInputOffsets under Forecast ' - 'section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ForecastInputOffsets under Forecast ' - 'section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper ForecastInputOffsets option specified in ' - 'the configuration file.') - if len(self.fcst_input_offsets) != self.number_inputs: - err_handler.err_out_screen('Please specify ForecastInputOffset values for each ' - 'corresponding input forcings for forecasts.') - # Check to make sure the input offset options make sense. There will be additional - # checking later when input choices are mapped to input products. - for inputOffset in self.fcst_input_offsets: - if inputOffset < 0: - err_handler.err_out_screen( - 'Please specify ForecastInputOffset values greater than or equal to zero.') - # Calculate the length of the forecast cycle, based on the maximum # length of the input forcing length chosen by the user. self.cycle_length_minutes = max(self.fcst_input_horizons) @@ -549,21 +825,28 @@ def read_config(self): 'maximum of the forecast time horizons specified.') # Calculate the number of output time steps per forecast cycle. self.num_output_steps = int(self.cycle_length_minutes / self.output_freq) - self.actual_output_steps = self.num_output_steps + if self.ana_flag: + self.actual_output_steps = np.int32(self.nFcsts) + else: + self.actual_output_steps = np.int32(self.num_output_steps) # Process geospatial information + try: - self.geogrid = config['Geospatial']['GeogridIn'] + geospatial = config['Geospatial'] + except KeyError: + err_handler.err_out_screen('Unable to locate Geospatial map in configuration file.') + + try: + self.geogrid = geospatial['GeogridIn'] except KeyError: - err_handler.err_out_screen('Unable to locate GeogridIn in the configuration file.') - except configparser.NoOptionError: err_handler.err_out_screen('Unable to locate GeogridIn in the configuration file.') if not os.path.isfile(self.geogrid): err_handler.err_out_screen('Unable to locate necessary geogrid file: ' + self.geogrid) # Check for the optional geospatial land metadata file. try: - self.spatial_meta = config['Geospatial']['SpatialMetaIn'] + self.spatial_meta = geospatial['SpatialMetaIn'] except KeyError: err_handler.err_out_screen('Unable to locate SpatialMetaIn in the configuration file.') if len(self.spatial_meta) == 0: @@ -573,46 +856,25 @@ def read_config(self): if not os.path.isfile(self.spatial_meta): err_handler.err_out_screen('Unable to locate optional spatial metadata file: ' + self.spatial_meta) - - # Check for the IgnoredBorderWidths + # Check for the optional grid metadata file. try: - self.ignored_border_widths = json.loads(config['Geospatial']['IgnoredBorderWidths']) - except (KeyError, configparser.NoOptionError): - # if didn't specify, no worries, just set to 0 - self.ignored_border_widths = [0.0]*self.number_inputs - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper IgnoredBorderWidths option specified in the configuration file.' - '({} was supplied'.format(config['Geospatial']['IgnoredBorderWidths'])) - if len(self.ignored_border_widths) != self.number_inputs: - err_handler.err_out_screen('Please specify IgnoredBorderWidths values for each ' - 'corresponding input forcings for SuppForcing.' - '({} was supplied'.format(self.ignored_border_widths)) - if any(map(lambda x: x < 0, self.ignored_border_widths)): - err_handler.err_out_screen('Please specify IgnoredBorderWidths values greater than or equal to zero:' - '({} was supplied'.format(self.ignored_border_widths)) + self.grid_meta = config['Geospatial'].get('GridMeta', '') + except KeyError: + err_handler.err_out_screen('Unable to locate Geospatial section in the configuration file.') + if len(self.grid_meta) == 0: + # No spatial metadata file found. + self.grid_meta = None + else: + if not os.path.isfile(self.grid_meta): + err_handler.err_out_screen('Unable to locate optional grid metadata file: ' + self.grid_meta) - # Process regridding options. try: - self.regrid_opt = json.loads(config['Regridding']['RegridOpt']) + regridding = config.get('Regridding') except KeyError: - err_handler.err_out_screen('Unable to locate RegridOpt under the Regridding section ' - 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RegridOpt under the Regridding section ' - 'in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper RegridOpt options specified in the configuration file.') - if len(self.regrid_opt) != self.number_inputs: - err_handler.err_out_screen('Please specify RegridOpt values for each corresponding input ' - 'forcings in the configuration file.') - # Check to make sure regridding options makes sense. - for regridOpt in self.regrid_opt: - if regridOpt < 1 or regridOpt > 3: - err_handler.err_out_screen('Invalid RegridOpt chosen in the configuration file. Please choose a ' - 'value of 1-3 for each corresponding input forcing.') + err_handler.err_out_screen('Unable to locate Regridding map in configuration file.') # Read weight file directory (optional) - self.weightsDir = config['Regridding'].get('RegridWeightsDir') + self.weightsDir = regridding.get('WeightsDir') if self.weightsDir is not None: # if we do have one specified, make sure it exists if not os.path.exists(self.weightsDir): @@ -623,401 +885,28 @@ def read_config(self): if self.realtime_flag: time_handling.calculate_lookback_window(self) - # Read in temporal interpolation options. - try: - self.forceTemoralInterp = json.loads(config['Interpolation']['ForcingTemporalInterpolation']) - except KeyError: - err_handler.err_out_screen('Unable to locate ForcingTemporalInterpolation under the Interpolation ' - 'section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ForcingTemporalInterpolation under the Interpolation ' - 'section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper ForcingTemporalInterpolation options specified in the ' - 'configuration file.') - if len(self.forceTemoralInterp) != self.number_inputs: - err_handler.err_out_screen('Please specify ForcingTemporalInterpolation values for each ' - 'corresponding input forcings in the configuration file.') - # Ensure the forcingTemporalInterpolation values make sense. - for temporalInterpOpt in self.forceTemoralInterp: - if temporalInterpOpt < 0 or temporalInterpOpt > 2: - err_handler.err_out_screen('Invalid ForcingTemporalInterpolation chosen in the configuration file. ' - 'Please choose a value of 0-2 for each corresponding input forcing.') - - # Read in the temperature downscaling options. - # Create temporary array to hold flags of if we need input parameter files. - param_flag = np.empty([len(self.input_forcings)], np.int) - param_flag[:] = 0 - try: - self.t2dDownscaleOpt = json.loads(config['Downscaling']['TemperatureDownscaling']) - except KeyError: - err_handler.err_out_screen('Unable to locate TemperatureDownscaling under the Downscaling ' - 'section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate TemperatureDownscaling under the Downscaling ' - 'section of the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper TemperatureDownscaling options specified in the configuration file.') - if len(self.t2dDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify TemperatureDownscaling values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the downscaling options chosen make sense. - count_tmp = 0 - for optTmp in self.t2dDownscaleOpt: - if optTmp < 0 or optTmp > 2: - err_handler.err_out_screen('Invalid TemperatureDownscaling options specified in ' - 'the configuration file.') - if optTmp == 2: - param_flag[count_tmp] = 1 - count_tmp = count_tmp + 1 - - # Read in the pressure downscaling options. - try: - self.psfcDownscaleOpt = json.loads(config['Downscaling']['PressureDownscaling']) - except KeyError: - err_handler.err_out_screen('Unable to locate PressureDownscaling under the Downscaling ' - 'section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate PressureDownscaling under the Downscaling ' - 'section of the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper PressureDownscaling options specified in the configuration file.') - if len(self.psfcDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify PressureDownscaling values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the downscaling options chosen make sense. - for optTmp in self.psfcDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid PressureDownscaling options specified in the configuration file.') - - # Read in the shortwave downscaling options - try: - self.swDownscaleOpt = json.loads(config['Downscaling']['ShortwaveDownscaling']) - except KeyError: - err_handler.err_out_screen('Unable to locate ShortwaveDownscaling under the Downscaling ' - 'section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate ShortwaveDownscaling under the Downscaling ' - 'section of the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper ShortwaveDownscaling options specified in the configuration file.') - if len(self.swDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify ShortwaveDownscaling values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the downscaling options chosen make sense. - for optTmp in self.swDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid ShortwaveDownscaling options specified in the configuration file.') - - # Read in the precipitation downscaling options - try: - self.precipDownscaleOpt = json.loads(config['Downscaling']['PrecipDownscaling']) - except KeyError: - err_handler.err_out_screen('Unable to locate PrecipDownscaling under the Downscaling ' - 'section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate PrecipDownscaling under the Downscaling ' - 'section of the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper PrecipDownscaling options specified in the configuration file.') - if len(self.precipDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify PrecipDownscaling values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the downscaling options chosen make sense. - count_tmp = 0 - for optTmp in self.precipDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid PrecipDownscaling options specified in the configuration file.') - if optTmp == 1: - param_flag[count_tmp] = 1 - count_tmp = count_tmp + 1 - - # Read in humidity downscaling options. - try: - self.q2dDownscaleOpt = json.loads(config['Downscaling']['HumidityDownscaling']) - except KeyError: - err_handler.err_out_screen('Unable to locate HumidityDownscaling under the Downscaling ' - 'section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate HumidityDownscaling under the Downscaling ' - 'section of the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper HumidityDownscaling options specified in the configuration file.') - if len(self.q2dDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify HumidityDownscaling values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the downscaling options chosen make sense. - for optTmp in self.q2dDownscaleOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid HumidityDownscaling options specified in the configuration file.') - - # Read in the downscaling parameter directory. - self.paramFlagArray = param_flag - tmp_scale_param_dirs = [] - if param_flag.sum() > 0: - self.paramFlagArray = param_flag - try: - tmp_scale_param_dirs = config.get('Downscaling', 'DownscalingParamDirs').split(',') - except KeyError: - err_handler.err_out_screen('Unable to locate DownscalingParamDirs in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate DownscalingParamDirs in the configuration file.') - if len(tmp_scale_param_dirs) == 1: - # single directory for all params - tmp_scale_param_dirs *= param_flag.sum() - elif len(tmp_scale_param_dirs) != param_flag.sum(): - err_handler.err_out_screen('Please specify a downscaling parameter directory for each ' - 'corresponding downscaling option that requires one.') - # Loop through each downscaling parameter directory and make sure they exist. - for dirTmp in range(0, len(tmp_scale_param_dirs)): - tmp_scale_param_dirs[dirTmp] = tmp_scale_param_dirs[dirTmp].strip() - if not os.path.isdir(tmp_scale_param_dirs[dirTmp]): - err_handler.err_out_screen('Unable to locate parameter directory: ' + tmp_scale_param_dirs[dirTmp]) - - # Create a list of downscaling parameter directories for each corresponding - # input forcing. If no directory is needed, or specified, we will set the value to NONE - self.dScaleParamDirs = [] - for count_tmp, _ in enumerate(self.input_forcings): - if param_flag[count_tmp] == 0: - self.dScaleParamDirs.append('NONE') - if param_flag[count_tmp] == 1: - self.dScaleParamDirs.append(tmp_scale_param_dirs[count_tmp]) - - # if the directory was specified but not downscaling, set it anyway for bias correction etc. - try: - if param_flag.sum() == 0 and len(config.get('Downscaling', 'DownscalingParamDirs').split(',')) == 1: - self.dScaleParamDirs = [config.get('Downscaling', 'DownscalingParamDirs').split(',')[0]] - except KeyError: - pass # TODO: this should not be `pass` if we have a parameter-based Bias Correction scheme selected - - # * Bias Correction Options * - - # Read AnA flag option - try: - # check both the Forecast section and if it's not there, the old BiasCorrection location - self.ana_flag = config['Forecast'].get('AnAFlag', config['BiasCorrection'].get('AnAFlag')) - if self.ana_flag is None: - raise KeyError - else: - self.ana_flag = int(self.ana_flag) - except KeyError: - err_handler.err_out_screen('Unable to locate AnAFlag in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate AnAFlag in the configuration file.') - except ValueError: - err_handler.err_out_screen('Improper AnAFlag value ') - if self.ana_flag < 0 or self.ana_flag > 1: - err_handler.err_out_screen('Please choose a AnAFlag value of 0 or 1.') - - # Read in temperature bias correction options - try: - self.t2BiasCorrectOpt = json.loads(config['BiasCorrection']['TemperatureBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate TemperatureBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate TemperatureBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper TemperatureBiasCorrection options specified in ' - 'the configuration file.') - if len(self.t2BiasCorrectOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify TemperatureBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.t2BiasCorrectOpt: - if optTmp < 0 or optTmp > 4: - err_handler.err_out_screen('Invalid TemperatureBiasCorrection options specified in the ' - 'configuration file.') - - # Read in surface pressure bias correction options. - try: - self.psfcBiasCorrectOpt = json.loads(config['BiasCorrection']['PressureBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate PressureBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate PressureBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper PressureBiasCorrection options specified in the configuration file.') - if len(self.psfcDownscaleOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify PressureBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.psfcBiasCorrectOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid PressureBiasCorrection options specified in the ' - 'configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in humidity bias correction options. - try: - self.q2BiasCorrectOpt = json.loads(config['BiasCorrection']['HumidityBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate HumidityBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate HumidityBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper HumdityBiasCorrection options specified in the configuration file.') - if len(self.q2BiasCorrectOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify HumidityBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.q2BiasCorrectOpt: - if optTmp < 0 or optTmp > 2: - err_handler.err_out_screen('Invalid HumidityBiasCorrection options specified in the ' - 'configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in wind bias correction options. - try: - self.windBiasCorrect = json.loads(config['BiasCorrection']['WindBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate WindBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate WindBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper WindBiasCorrection options specified in the configuration file.') - if len(self.windBiasCorrect) != self.number_inputs: - err_handler.err_out_screen('Please specify WindBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.windBiasCorrect: - if optTmp < 0 or optTmp > 4: - err_handler.err_out_screen('Invalid WindBiasCorrection options specified in the configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in shortwave radiation bias correction options. - try: - self.swBiasCorrectOpt = json.loads(config['BiasCorrection']['SwBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate SwBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SwBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper SwBiasCorrection options specified in the configuration file.') - if len(self.swBiasCorrectOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify SwBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.swBiasCorrectOpt: - if optTmp < 0 or optTmp > 2: - err_handler.err_out_screen('Invalid SwBiasCorrection options specified in the configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in longwave radiation bias correction options. - try: - self.lwBiasCorrectOpt = json.loads(config['BiasCorrection']['LwBiasCorrection']) - except KeyError: - err_handler.err_out_screen('Unable to locate LwBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate LwBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper LwBiasCorrection options specified in ' - 'the configuration file.') - if len(self.lwBiasCorrectOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify LwBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.lwBiasCorrectOpt: - if optTmp < 0 or optTmp > 4: - err_handler.err_out_screen('Invalid LwBiasCorrection options specified in the configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True - - # Read in precipitation bias correction options. try: - self.precipBiasCorrectOpt = json.loads(config['BiasCorrection']['PrecipBiasCorrection']) + suppforcings = config['SuppForcing'] except KeyError: - err_handler.err_out_screen('Unable to locate PrecipBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate PrecipBiasCorrection under the ' - 'BiasCorrection section of the configuration file.') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper PrecipBiasCorrection options specified in the configuration file.') - if len(self.precipBiasCorrectOpt) != self.number_inputs: - err_handler.err_out_screen('Please specify PrecipBiasCorrection values for each corresponding ' - 'input forcings in the configuration file.') - # Ensure the bias correction options chosen make sense. - for optTmp in self.precipBiasCorrectOpt: - if optTmp < 0 or optTmp > 1: - err_handler.err_out_screen('Invalid PrecipBiasCorrection options specified in the configuration file.') - if optTmp == 1: - # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. - self.runCfsNldasBiasCorrect = True + err_handler.err_out_screen('Unable to locate SuppForcing map in configuration file.') - # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on - # for ALL variables. - if self.runCfsNldasBiasCorrect: - if min(self.precipBiasCorrectOpt) != 1 and max(self.precipBiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'Precipitation under this configuration.') - if min(self.lwBiasCorrectOpt) != 1 and max(self.lwBiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'long-wave radiation under this configuration.') - if min(self.swBiasCorrectOpt) != 1 and max(self.swBiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'short-wave radiation under this configuration.') - if min(self.t2BiasCorrectOpt) != 1 and max(self.t2BiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'surface temperature under this configuration.') - if min(self.windBiasCorrect) != 1 and max(self.windBiasCorrect) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'wind forcings under this configuration.') - if min(self.q2BiasCorrectOpt) != 1 and max(self.q2BiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'specific humidity under this configuration.') - if min(self.psfcBiasCorrectOpt) != 1 and max(self.psfcBiasCorrectOpt) != 1: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' - 'surface pressure under this configuration.') - # Make sure we don't have any other forcings activated. This can only be ran for CFSv2. - for optTmp in self.input_forcings: - if optTmp != 7: - err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction can only be used in ' - 'CFSv2-only configurations') # Read in supplemental precipitation options as an array of values to map. try: - self.supp_precip_forcings = json.loads(config['SuppForcing']['SuppPcp']) + self.supp_precip_forcings = [suppforcing['Pcp'] for suppforcing in suppforcings] except KeyError: - err_handler.err_out_screen('Unable to locate SuppPcp under SuppForcing section in configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcp under SuppForcing section in configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper SuppPcp option specified in configuration file') + err_handler.err_out_screen('Please pick SuppForcing[i][\'Pcp\'] from options: %s' % [str(item) for item in self.SuppForcingPcpEnum]) self.number_supp_pcp = len(self.supp_precip_forcings) # Read in the supp pcp types (GRIB[1|2], NETCDF) try: - self.supp_precip_file_types = config.get('SuppForcing', 'SuppPcpForcingTypes').strip("[]").split(',') + self.supp_precip_file_types = [suppforcing['PcpType'] for suppforcing in suppforcings] self.supp_precip_file_types = [stype.strip() for stype in self.supp_precip_file_types] if self.supp_precip_file_types == ['']: self.supp_precip_file_types = [] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpForcingTypes in SuppForcing section ' 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpForcingTypes in SuppForcing section ' - 'in the configuration file.') if len(self.supp_precip_file_types) != self.number_supp_pcp: err_handler.err_out_screen('Number of SuppPcpForcingTypes ({}) must match the number ' 'of SuppPcp inputs ({}) in the configuration file.'.format(len(self.supp_precip_file_types), self.number_supp_pcp)) @@ -1030,21 +919,12 @@ def read_config(self): # Check to make sure supplemental precip options make sense. Also read in the RQI threshold # if any radar products where chosen. for suppOpt in self.supp_precip_forcings: - if suppOpt < 0 or suppOpt > 11: - err_handler.err_out_screen('Please specify SuppForcing values between 1 and 11.') # Read in RQI threshold to apply to radar products. - if suppOpt in (1,2,7,10,11): + if suppOpt in ('MRMS','MRMS_GAGE','MRMS_SBCV2','AK_MRMS','AK_NWS_IV'): try: - self.rqiMethod = json.loads(config['SuppForcing']['RqiMethod']) + self.rqiMethod = [suppforcing['RqiMethod'] for suppforcing in suppforcings] except KeyError: - err_handler.err_out_screen('Unable to locate RqiMethod under SuppForcing ' - 'section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RqiMethod under SuppForcing ' - 'section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper RqiMethod option in the configuration file.') - + err_handler.err_out_screen('Please pick SuppForcing[i][\'RqiMethod\'] from options: %s' % [str(item) for item in SuppForcingRqiMethodEnum]) # Check that if we have more than one RqiMethod, it's the correct number if type(self.rqiMethod) is list: if len(self.rqiMethod) != self.number_supp_pcp: @@ -1052,25 +932,15 @@ def read_config(self): 'of SuppPcp inputs ({}) in the configuration file, or ' 'supply a single method for all inputs'.format( len(self.rqiMethod), self.number_supp_pcp)) - elif type(self.rqiMethod) is int: + elif type(self.rqiMethod) is str: # Support 'classic' mode of single method self.rqiMethod = [self.rqiMethod] * self.number_supp_pcp - # Make sure the RqiMethod(s) makes sense. - for method in self.rqiMethod: - if method < 0 or method > 2: - err_handler.err_out_screen('Please specify RqiMethods of either 0, 1, or 2.') - try: - self.rqiThresh = json.loads(config['SuppForcing']['RqiThreshold']) + self.rqiThresh = [suppforcing['RqiThreshold'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate RqiThreshold under ' - 'SuppForcing section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RqiThreshold under ' - 'SuppForcing section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper RqiThreshold option in the configuration file.') + 'SuppForcing section in thelf.supp_precip_param_dir configuration file.') # Check that if we have more than one RqiThreshold, it's the correct number if type(self.rqiThresh) is list: @@ -1090,14 +960,10 @@ def read_config(self): # Read in the input directories for each supplemental precipitation product. try: - self.supp_precip_dirs = config.get('SuppForcing', 'SuppPcpDirectories').split(',') + self.supp_precip_dirs = [suppforcing['PcpDir'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section ' 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section ' - 'in the configuration file.') - # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace # or new line characters. for dirTmp in range(0, len(self.supp_precip_dirs)): @@ -1106,7 +972,7 @@ def read_config(self): err_handler.err_out_screen('Unable to locate supp pcp directory: ' + self.supp_precip_dirs[dirTmp]) #Special case for ExtAnA where we treat comma separated stage IV, MRMS data as one SuppPcp input - if 11 in self.supp_precip_forcings: + if 'AK_NWS_IV' in self.supp_precip_forcings: if len(self.supp_precip_forcings) != 1: err_handler.err_out_screen('Alaska Stage IV/MRMS SuppPcp option is only supported as a standalone option') self.supp_precip_dirs = [",".join(self.supp_precip_dirs)] @@ -1117,15 +983,10 @@ def read_config(self): # Process supplemental precipitation enforcement options try: - self.supp_precip_mandatory = json.loads(config['SuppForcing']['SuppPcpMandatory']) + self.supp_precip_mandatory = [int(suppforcing['PcpMandatory']) for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpMandatory under the SuppForcing section ' 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpMandatory under the SuppForcing section ' - 'in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper SuppPcpMandatory options specified in the configuration file.') if len(self.supp_precip_mandatory) != self.number_supp_pcp: err_handler.err_out_screen('Please specify SuppPcpMandatory values for each corresponding ' 'supplemental precipitation options in the configuration file.') @@ -1138,15 +999,10 @@ def read_config(self): # Read in the regridding options. try: - self.regrid_opt_supp_pcp = json.loads(config['SuppForcing']['RegridOptSuppPcp']) + self.regrid_opt_supp_pcp = [suppforcing['RegridOptPcp'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate RegridOptSuppPcp under the SuppForcing section ' 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate RegridOptSuppPcp under the SuppForcing section ' - 'in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper RegridOptSuppPcp options specified in the configuration file.') if len(self.regrid_opt_supp_pcp) != self.number_supp_pcp: err_handler.err_out_screen('Please specify RegridOptSuppPcp values for each corresponding supplemental ' 'precipitation product in the configuration file.') @@ -1159,16 +1015,10 @@ def read_config(self): # Read in temporal interpolation options. try: - self.suppTemporalInterp = json.loads(config['SuppForcing']['SuppPcpTemporalInterpolation']) + self.suppTemporalInterp = [suppforcing['PcpTemporalInterp'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpTemporalInterpolation under the SuppForcing ' 'section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpTemporalInterpolation under the SuppForcing ' - 'section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper SuppPcpTemporalInterpolation options specified in the ' - 'configuration file.') if len(self.suppTemporalInterp) != self.number_supp_pcp: err_handler.err_out_screen('Please specify SuppPcpTemporalInterpolation values for each ' 'corresponding supplemental precip products in the configuration file.') @@ -1180,13 +1030,10 @@ def read_config(self): # Read in max time option try: - self.supp_pcp_max_hours = json.loads(config['SuppForcing']['SuppPcpMaxHours']) - except (KeyError, configparser.NoOptionError): + self.supp_pcp_max_hours = [suppforcing['PcpMaxHours'] for suppforcing in suppforcings] + except (KeyError): self.supp_pcp_max_hours = None # if missing, don't care, just assume all time - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper SuppPcpMaxHours options specified in the ' - 'configuration file.') if type(self.supp_pcp_max_hours) is list: if len(self.supp_pcp_max_hours) != self.number_supp_pcp: @@ -1200,16 +1047,10 @@ def read_config(self): # Read in the SuppPcpInputOffsets options. try: - self.supp_input_offsets = json.loads(config['SuppForcing']['SuppPcpInputOffsets']) + self.supp_input_offsets = [suppforcing['PcpInputOffsets'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpInputOffsets under SuppForcing ' 'section in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpInputOffsets under SuppForcing ' - 'section in the configuration file.') - except json.decoder.JSONDecodeError: - err_handler.err_out_screen('Improper SuppPcpInputOffsets option specified in ' - 'the configuration file.') if len(self.supp_input_offsets) != self.number_supp_pcp: err_handler.err_out_screen('Please specify SuppPcpInputOffsets values for each ' 'corresponding input forcings for SuppForcing.') @@ -1222,59 +1063,32 @@ def read_config(self): # Read in the optional parameter directory for supplemental precipitation. try: - self.supp_precip_param_dir = config['SuppForcing']['SuppPcpParamDir'] + self.supp_precip_param_dir = [suppforcing['PcpParamDir'] for suppforcing in suppforcings] except KeyError: err_handler.err_out_screen('Unable to locate SuppPcpParamDir under the SuppForcing section ' 'in the configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate SuppPcpParamDir under the SuppForcing section ' - 'in the configuration file.') - except ValueError: - err_handler.err_out_screen('Improper SuppPcpParamDir option specified in the configuration file.') - if not os.path.isdir(self.supp_precip_param_dir): - err_handler.err_out_screen('Unable to locate SuppPcpParamDir: ' + self.supp_precip_param_dir) - + for dir in self.supp_precip_param_dir: + if not os.path.isdir(dir): + err_handler.err_out_screen('Unable to locate SuppForcing[i][\'PcpParamDir\']: ' + dir) + #For compatability only keep the first PcpParamDir + self.supp_precip_param_dir = self.supp_precip_param_dir[0] # Read in Ensemble information # Read in CFS ensemble member information IF we have chosen CFSv2 as an input # forcing. for optTmp in self.input_forcings: - if optTmp == 7: + if optTmp == 'CFS_V2': try: - self.cfsv2EnsMember = json.loads(config['Ensembles']['cfsEnsNumber']) + self.cfsv2EnsMember = [input['Ensembles']['cfsEnsNumber'] for input in inputs if 'cfsEnsNumber' in input['Ensembles']][0] except KeyError: err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles ' 'section of the configuration file') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles ' - 'section of the configuration file') - except json.JSONDecodeError: - err_handler.err_out_screen('Improper cfsEnsNumber options specified in the configuration file') if self.cfsv2EnsMember < 1 or self.cfsv2EnsMember > 4: err_handler.err_out_screen('Please chose an cfsEnsNumber value of 1,2,3 or 4.') - # Read in information for the custom input NetCDF files that are to be processed. - # Read in the ForecastInputHorizons options. - try: - self.customFcstFreq = json.loads(config['Custom']['custom_input_fcst_freq']) - except KeyError: - err_handler.err_out_screen('Unable to locate custom_input_fcst_freq under Custom section in ' - 'configuration file.') - except configparser.NoOptionError: - err_handler.err_out_screen('Unable to locate custom_input_fcst_freq under Custom section in ' - 'configuration file.') - except json.decoder.JSONDecodeError as je: - err_handler.err_out_screen('Improper custom_input_fcst_freq option specified in ' - 'configuration file: ' + str(je)) - if len(self.customFcstFreq) != self.number_custom_inputs: - err_handler.err_out_screen(f'Improper custom_input fcst_freq specified. ' - f'This number ({len(self.customFcstFreq)}) must ' - f'match the frequency of custom input forcings selected ' - f'({self.number_custom_inputs}).') - @property def use_data_at_current_time(self): if self.supp_pcp_max_hours is not None: hrs_since_start = self.current_output_date - self.current_fcst_cycle return hrs_since_start <= datetime.timedelta(hours = self.supp_pcp_max_hours) else: - return True + return True diff --git a/core/config_v1.py b/core/config_v1.py new file mode 100755 index 0000000..b1d4524 --- /dev/null +++ b/core/config_v1.py @@ -0,0 +1,1250 @@ +#!/usr/bin/env python + +import configparser +import datetime +import json +import os +import sys +import numpy as np +from core import err_handler +import math + +class ConfigOptions: + """ + Configuration abstract class for configuration options read in from the file + specified by the user. + """ + + def __init__(self, config): + """ + Initialize the configuration class to empty None attributes + param config: The user-specified path to the configuration file. + """ + self.input_forcings = None + self.supp_precip_forcings = None + self.input_force_dirs = None + self.input_force_types = None + self.supp_precip_dirs = None + self.supp_precip_file_types = None + self.supp_precip_param_dir = None + self.input_force_mandatory = None + self.supp_precip_mandatory = None + self.number_inputs = None + self.number_supp_pcp = None + self.number_custom_inputs = 0 + self.output_freq = None + self.output_dir = None + self.scratch_dir = None + self.useCompression = 0 + self.useFloats = 0 + self.num_output_steps = None + self.retro_flag = None + self.realtime_flag = None + self.refcst_flag = None + self.ana_flag = None + self.ana_out_dir = None + self.b_date_proc = None + self.e_date_proc = None + self.first_fcst_cycle = None + self.current_fcst_cycle = None + self.current_output_step = None + self.cycle_length_minutes = None + self.prev_output_date = None + self.current_output_date = None + self.look_back = None + self.fcst_freq = None + self.nFcsts = None + self.fcst_shift = None + self.fcst_input_horizons = None + self.fcst_input_offsets = None + self.process_window = None + self.geogrid = None + self.spatial_meta = None + self.ignored_border_widths = None + self.regrid_opt = None + self.weightsDir = None + self.regrid_opt_supp_pcp = None + self.config_path = config + self.errMsg = None + self.statusMsg = None + self.logFile = None + self.logHandle = None + self.dScaleParamDirs = None + self.paramFlagArray = None + self.forceTemoralInterp = None + self.suppTemporalInterp = None + self.t2dDownscaleOpt = None + self.swDownscaleOpt = None + self.psfcDownscaleOpt = None + self.precipDownscaleOpt = None + self.q2dDownscaleOpt = None + self.t2BiasCorrectOpt = None + self.psfcBiasCorrectOpt = None + self.q2BiasCorrectOpt = None + self.windBiasCorrect = None + self.swBiasCorrectOpt = None + self.lwBiasCorrectOpt = None + self.precipBiasCorrectOpt = None + self.runCfsNldasBiasCorrect = False + self.cfsv2EnsMember = None + self.customFcstFreq = None + self.rqiMethod = None + self.rqiThresh = 1.0 + self.globalNdv = -9999.0 + self.d_program_init = datetime.datetime.utcnow() + self.errFlag = 0 + self.nwmVersion = None + self.nwmConfig = None + + def read_config(self): + """ + Read in options from the configuration file and check that proper options + were provided. + """ + # Read in the configuration file + config = configparser.ConfigParser() + try: + config.read(self.config_path) + except KeyError: + err_handler.err_out_screen('Unable to open the configuration file: ' + self.config_path) + + # Read in the base input forcing options as an array of values to map. + try: + self.input_forcings = json.loads(config['Input']['InputForcings']) + except KeyError: + err_handler.err_out_screen('Unable to locate InputForcings under Input section in configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate InputForcings under Input section in configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper InputForcings option specified in configuration file') + if len(self.input_forcings) == 0: + err_handler.err_out_screen('Please choose at least one InputForcings dataset to process') + self.number_inputs = len(self.input_forcings) + + # Check to make sure forcing options make sense + for forceOpt in self.input_forcings: + if forceOpt < 0 or forceOpt > 21: + err_handler.err_out_screen('Please specify InputForcings values between 1 and 18.') + # Keep tabs on how many custom input forcings we have. + if forceOpt == 10: + self.number_custom_inputs = self.number_custom_inputs + 1 + + # Read in the input forcings types (GRIB[1|2], NETCDF) + try: + self.input_force_types = config.get('Input', 'InputForcingTypes').strip("[]").split(',') + self.input_force_types = [ftype.strip() for ftype in self.input_force_types] + if self.input_force_types == ['']: + self.input_force_types = [] + except KeyError: + err_handler.err_out_screen('Unable to locate InputForcingTypes in Input section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate InputForcingTypes in Input section ' + 'in the configuration file.') + if len(self.input_force_types) != self.number_inputs: + err_handler.err_out_screen('Number of InputForcingTypes must match the number ' + 'of InputForcings in the configuration file.') + for fileType in self.input_force_types: + if fileType not in ['GRIB1', 'GRIB2', 'NETCDF']: + err_handler.err_out_screen('Invalid forcing file type "{}" specified. ' + 'Only GRIB1, GRIB2, and NETCDF are supported'.format(fileType)) + + # Read in the input directories for each forcing option. + try: + self.input_force_dirs = config.get('Input', 'InputForcingDirectories').split(',') + except KeyError: + err_handler.err_out_screen('Unable to locate InputForcingDirectories in Input section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate InputForcingDirectories in Input section ' + 'in the configuration file.') + if len(self.input_force_dirs) != self.number_inputs: + err_handler.err_out_screen('Number of InputForcingDirectories must match the number ' + 'of InputForcings in the configuration file.') + # Loop through and ensure all input directories exist. Also strip out any whitespace + # or new line characters. + for dirTmp in range(0, len(self.input_force_dirs)): + self.input_force_dirs[dirTmp] = self.input_force_dirs[dirTmp].strip() + if not os.path.isdir(self.input_force_dirs[dirTmp]): + err_handler.err_out_screen('Unable to locate forcing directory: ' + + self.input_force_dirs[dirTmp]) + + # Read in the mandatory enforcement options for input forcings. + try: + self.input_force_mandatory = json.loads(config['Input']['InputMandatory']) + except KeyError: + err_handler.err_out_screen('Unable to locate InputMandatory under Input section in configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate InputMandatory under Input section in configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper InputMandatory option specified in configuration file') + + # Process input forcing enforcement options + try: + self.input_force_mandatory = json.loads(config['Input']['InputMandatory']) + except KeyError: + err_handler.err_out_screen('Unable to locate InputMandatory under the Input section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate InputMandatory under the Input section ' + 'in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper InputMandatory options specified in the configuration file.') + if len(self.input_force_mandatory) != self.number_inputs: + err_handler.err_out_screen('Please specify InputMandatory values for each corresponding input ' + 'forcings in the configuration file.') + # Check to make sure enforcement options makes sense. + for enforceOpt in self.input_force_mandatory: + if enforceOpt < 0 or enforceOpt > 1: + err_handler.err_out_screen('Invalid InputMandatory chosen in the configuration file. Please' + ' choose a value of 0 or 1 for each corresponding input forcing.') + + # Read in the output frequency + try: + self.output_freq = int(config['Output']['OutputFrequency']) + except ValueError: + err_handler.err_out_screen('Improper OutputFrequency value specified in the configuration file.') + except KeyError: + err_handler.err_out_screen('Unable to locate OutputFrequency in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate OutputFrequency in the configuration file.') + if self.output_freq <= 0: + err_handler.err_out_screen('Please specify an OutputFrequency that is greater than zero minutes.') + + # Read in the output directory + try: + self.output_dir = config['Output']['OutDir'] + except ValueError: + err_handler.err_out_screen('Improper OutDir specified in the configuration file.') + except KeyError: + err_handler.err_out_screen('Unable to locate OutDir in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate OutDir in the configuration file.') + if not os.path.isdir(self.output_dir): + err_handler.err_out_screen('Specified output directory: ' + self.output_dir + ' not found.') + + # Read in the scratch temporary directory. + try: + self.scratch_dir = config['Output']['ScratchDir'] + except ValueError: + err_handler.err_out_screen('Improper ScratchDir specified in the configuration file.') + except KeyError: + err_handler.err_out_screen('Unable to locate ScratchDir in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ScratchDir in the configuration file.') + if not os.path.isdir(self.scratch_dir): + err_handler.err_out_screen('Specified output directory: ' + self.scratch_dir + ' not found') + + # Read in compression option + try: + self.useCompression = int(config['Output']['compressOutput']) + except KeyError: + err_handler.err_out_screen('Unable to locate compressOut in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate compressOut in the configuration file.') + except ValueError: + err_handler.err_out_screen('Improper compressOut value.') + if self.useCompression < 0 or self.useCompression > 1: + err_handler.err_out_screen('Please choose a compressOut value of 0 or 1.') + + # Read in floating-point option + try: + self.useFloats = int(config['Output']['floatOutput']) + except KeyError: + # err_handler.err_out_screen('Unable to locate floatOutput in the configuration file.') + self.useFloats = 0 + except configparser.NoOptionError: + # err_handler.err_out_screen('Unable to locate floatOutput in the configuration file.') + self.useFloats = 0 + except ValueError: + err_handler.err_out_screen('Improper floatOutput value: {}'.format(config['Output']['floatOutput'])) + if self.useFloats < 0 or self.useFloats > 1: + err_handler.err_out_screen('Please choose a floatOutput value of 0 or 1.') + + # Read in retrospective options + try: + self.retro_flag = int(config['Retrospective']['RetroFlag']) + except KeyError: + err_handler.err_out_screen('Unable to locate RetroFlag in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RetroFlag in the configuration file.') + except ValueError: + err_handler.err_out_screen('Improper RetroFlag value ') + if self.retro_flag < 0 or self.retro_flag > 1: + err_handler.err_out_screen('Please choose a RetroFlag value of 0 or 1.') + + # Process the beginning date of forcings to process. + if self.retro_flag == 1: + self.realtime_flag = False + self.refcst_flag = False + try: + beg_date_tmp = config['Retrospective']['BDateProc'] + except KeyError: + err_handler.err_out_screen('Unable to locate BDateProc under Logistics section in ' + 'configuration file.') + beg_date_tmp = None + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate BDateProc under Logistics section in ' + 'configuration file.') + beg_date_tmp = None + if beg_date_tmp != '-9999': + if len(beg_date_tmp) != 12: + err_handler.err_out_screen('Improper BDateProc length entered into the ' + 'configuration file. Please check your entry.') + try: + self.b_date_proc = datetime.datetime.strptime(beg_date_tmp, '%Y%m%d%H%M') + except ValueError: + err_handler.err_out_screen('Improper BDateProc value entered into the ' + 'configuration file. Please check your entry.') + else: + self.b_date_proc = -9999 + + # Process the ending date of retrospective forcings to process + try: + end_date_tmp = config['Retrospective']['EDateProc'] + except KeyError: + err_handler.err_out_screen('Unable to locate EDateProc under Logistics section in ' + 'configuration file.') + end_date_tmp = None + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate EDateProc under Logistics section in ' + 'configuration file.') + end_date_tmp = None + if end_date_tmp != '-9999': + if len(end_date_tmp) != 12: + err_handler.err_out_screen('Improper EDateProc length entered into the ' + 'configuration file. Please check your entry.') + try: + self.e_date_proc = datetime.datetime.strptime(end_date_tmp, '%Y%m%d%H%M') + except ValueError: + err_handler.err_out_screen('Improper EDateProc value entered into the ' + 'configuration file. Please check your entry.') + if self.b_date_proc == -9999 and self.e_date_proc != -9999: + err_handler.err_out_screen('If choosing retrospective forecasting, dates must not be -9999') + if self.e_date_proc <= self.b_date_proc: + err_handler.err_out_screen('Please choose an ending EDateProc that is greater than BDateProc.') + else: + self.e_date_proc = -9999 + if self.e_date_proc == -9999 and self.b_date_proc != -9999: + err_handler.err_out_screen('If choosing retrospective forcings, dates must not be -9999') + + # Calculate the number of output time steps + dt_tmp = self.e_date_proc - self.b_date_proc + self.num_output_steps = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.output_freq) + + # Process realtime or reforecasting options. + if self.retro_flag == 0: + # If the retro flag is off, we are assuming a realtime or reforecast simulation. + try: + self.look_back = int(config['Forecast']['LookBack']) + if self.look_back <= 0 and self.look_back != -9999: + err_handler.err_out_screen('Please specify a positive LookBack or -9999 for realtime.') + except ValueError: + err_handler.err_out_screen('Improper LookBack value entered into the ' + 'configuration file. Please check your entry.') + except KeyError: + err_handler.err_out_screen('Unable to locate LookBack in the configuration ' + 'file. Please verify entries exist.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate LookBack in the configuration ' + 'file. Please verify entries exist.') + + # Process the beginning date of reforecast forcings to process + try: + beg_date_tmp = config['Forecast']['RefcstBDateProc'] + except KeyError: + err_handler.err_out_screen('Unable to locate RefcstBDateProc under Logistics section in ' + 'configuration file.') + beg_date_tmp = None + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RefcstBDateProc under Logistics section in ' + 'configuration file.') + beg_date_tmp = None + if beg_date_tmp != '-9999': + if len(beg_date_tmp) != 12: + err_handler.err_out_screen('Improper RefcstBDateProc length entered into the ' + 'configuration file. Please check your entry.') + try: + self.b_date_proc = datetime.datetime.strptime(beg_date_tmp, '%Y%m%d%H%M') + except ValueError: + err_handler.err_out_screen('Improper RefcstBDateProc value entered into the ' + 'configuration file. Please check your entry.') + else: + self.b_date_proc = -9999 + + # Process the ending date of reforecast forcings to process + try: + end_date_tmp = config['Forecast']['RefcstEDateProc'] + except KeyError: + err_handler.err_out_screen('Unable to locate RefcstEDateProc under Logistics section in ' + 'configuration file.') + end_date_tmp = None + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RefcstEDateProc under Logistics section in ' + 'configuration file.') + end_date_tmp = None + if end_date_tmp != '-9999': + if len(end_date_tmp) != 12: + err_handler.err_out_screen('Improper RefcstEDateProc length entered into the' + 'configuration file. Please check your entry.') + try: + self.e_date_proc = datetime.datetime.strptime(end_date_tmp, '%Y%m%d%H%M') + except ValueError: + err_handler.err_out_screen('Improper RefcstEDateProc value entered into the ' + 'configuration file. Please check your entry.') + else: + self.e_date_proc = -9999 + + if self.e_date_proc != -9999 and self.e_date_proc <= self.b_date_proc: + err_handler.err_out_screen('Please choose an ending RefcstEDateProc that is greater ' + 'than RefcstBDateProc.') + + # If the Retro flag is off, and lookback is off, then we assume we are + # running a reforecast. + if self.look_back == -9999: + self.realtime_flag = False + self.refcst_flag = True + + elif self.b_date_proc == -9999 and self.e_date_proc == -9999: + self.realtime_flag = True + self.refcst_flag = True + + else: + # The processing window will be calculated based on current time and the + # lookback option since this is a realtime instance. + self.realtime_flag = False + self.refcst_flag = False + # self.b_date_proc = -9999 + # self.e_date_proc = -9999 + + # Calculate the delta time between the beginning and ending time of processing. + # self.process_window = self.e_date_proc - self.b_date_proc + + # Read in the ForecastFrequency option. + try: + self.fcst_freq = int(config['Forecast']['ForecastFrequency']) + except ValueError: + err_handler.err_out_screen('Improper ForecastFrequency value entered into ' + 'the configuration file. Please check your entry.') + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastFrequency in the configuration ' + 'file. Please verify entries exist.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ForecastFrequency in the configuration ' + 'file. Please verify entries exist.') + if self.fcst_freq <= 0: + err_handler.err_out_screen('Please specify a ForecastFrequency in the configuration ' + 'file greater than zero.') + # Currently, we only support daily or sub-daily forecasts. Any other iterations should + # be done using custom config files for each forecast cycle. + if self.fcst_freq > 1440: + err_handler.err_out_screen('Only forecast cycles of daily or sub-daily are supported ' + 'at this time') + + # Read in the ForecastShift option. This is ONLY done for the realtime instance as + # it's used to calculate the beginning of the processing window. + if True: # was: self.realtime_flag: + try: + self.fcst_shift = int(config['Forecast']['ForecastShift']) + except ValueError: + err_handler.err_out_screen('Improper ForecastShift value entered into the ' + 'configuration file. Please check your entry.') + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastShift in the configuration ' + 'file. Please verify entries exist.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ForecastShift in the configuration ' + 'file. Please verify entries exist.') + if self.fcst_shift < 0: + err_handler.err_out_screen('Please specify a ForecastShift in the configuration ' + 'file greater than or equal to zero.') + + # Calculate the beginning/ending processing dates if we are running realtime + if self.realtime_flag: + calculate_lookback_window(self) + + if self.refcst_flag: + # Calculate the number of forecasts to issue, and verify the user has chosen a + # correct divider based on the dates + dt_tmp = self.e_date_proc - self.b_date_proc + if (dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) % self.fcst_freq != 0: + err_handler.err_out_screen('Please choose an equal divider forecast frequency for your ' + 'specified reforecast range.') + self.nFcsts = int((dt_tmp.days * 1440 + dt_tmp.seconds / 60.0) / self.fcst_freq) + + if self.look_back != -9999: + calculate_lookback_window(self) + + # Read in the ForecastInputHorizons options. + try: + self.fcst_input_horizons = json.loads(config['Forecast']['ForecastInputHorizons']) + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastInputHorizons under Forecast section in ' + 'configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ForecastInputHorizons under Forecast section in ' + 'configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper ForecastInputHorizons option specified in ' + 'configuration file') + if len(self.fcst_input_horizons) != self.number_inputs: + err_handler.err_out_screen('Please specify ForecastInputHorizon values for ' + 'each corresponding input forcings for forecasts.') + + # Check to make sure the horizons options make sense. There will be additional + # checking later when input choices are mapped to input products. + for horizonOpt in self.fcst_input_horizons: + if horizonOpt <= 0: + err_handler.err_out_screen('Please specify ForecastInputHorizon values greater ' + 'than zero.') + + # Read in the ForecastInputOffsets options. + try: + self.fcst_input_offsets = json.loads(config['Forecast']['ForecastInputOffsets']) + except KeyError: + err_handler.err_out_screen('Unable to locate ForecastInputOffsets under Forecast ' + 'section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ForecastInputOffsets under Forecast ' + 'section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper ForecastInputOffsets option specified in ' + 'the configuration file.') + if len(self.fcst_input_offsets) != self.number_inputs: + err_handler.err_out_screen('Please specify ForecastInputOffset values for each ' + 'corresponding input forcings for forecasts.') + # Check to make sure the input offset options make sense. There will be additional + # checking later when input choices are mapped to input products. + for inputOffset in self.fcst_input_offsets: + if inputOffset < 0: + err_handler.err_out_screen( + 'Please specify ForecastInputOffset values greater than or equal to zero.') + + # Calculate the length of the forecast cycle, based on the maximum + # length of the input forcing length chosen by the user. + self.cycle_length_minutes = max(self.fcst_input_horizons) + + # Ensure the number maximum cycle length is an equal divider of the output + # time step specified by the user. + if self.cycle_length_minutes % self.output_freq != 0: + err_handler.err_out_screen('Please specify an output time step that is an equal divider of the ' + 'maximum of the forecast time horizons specified.') + # Calculate the number of output time steps per forecast cycle. + self.num_output_steps = int(self.cycle_length_minutes / self.output_freq) + + # Process geospatial information + try: + self.geogrid = config['Geospatial']['GeogridIn'] + except KeyError: + err_handler.err_out_screen('Unable to locate GeogridIn in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate GeogridIn in the configuration file.') + if not os.path.isfile(self.geogrid): + err_handler.err_out_screen('Unable to locate necessary geogrid file: ' + self.geogrid) + + # Check for the optional geospatial land metadata file. + try: + self.spatial_meta = config['Geospatial']['SpatialMetaIn'] + except KeyError: + err_handler.err_out_screen('Unable to locate SpatialMetaIn in the configuration file.') + if len(self.spatial_meta) == 0: + # No spatial metadata file found. + self.spatial_meta = None + else: + if not os.path.isfile(self.spatial_meta): + err_handler.err_out_screen('Unable to locate optional spatial metadata file: ' + + self.spatial_meta) + + # Check for the IgnoredBorderWidths + try: + self.ignored_border_widths = json.loads(config['Geospatial']['IgnoredBorderWidths']) + except (KeyError, configparser.NoOptionError): + # if didn't specify, no worries, just set to 0 + self.ignored_border_widths = [0.0]*self.number_inputs + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper IgnoredBorderWidths option specified in the configuration file.' + '({} was supplied'.format(config['Geospatial']['IgnoredBorderWidths'])) + if len(self.ignored_border_widths) != self.number_inputs: + err_handler.err_out_screen('Please specify IgnoredBorderWidths values for each ' + 'corresponding input forcings for SuppForcing.' + '({} was supplied'.format(self.ignored_border_widths)) + if any(map(lambda x: x < 0, self.ignored_border_widths)): + err_handler.err_out_screen('Please specify IgnoredBorderWidths values greater than or equal to zero:' + '({} was supplied'.format(self.ignored_border_widths)) + + # Process regridding options. + try: + self.regrid_opt = json.loads(config['Regridding']['RegridOpt']) + except KeyError: + err_handler.err_out_screen('Unable to locate RegridOpt under the Regridding section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RegridOpt under the Regridding section ' + 'in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper RegridOpt options specified in the configuration file.') + if len(self.regrid_opt) != self.number_inputs: + err_handler.err_out_screen('Please specify RegridOpt values for each corresponding input ' + 'forcings in the configuration file.') + # Check to make sure regridding options makes sense. + for regridOpt in self.regrid_opt: + if regridOpt < 1 or regridOpt > 3: + err_handler.err_out_screen('Invalid RegridOpt chosen in the configuration file. Please choose a ' + 'value of 1-3 for each corresponding input forcing.') + + # Read weight file directory (optional) + self.weightsDir = config['Regridding'].get('RegridWeightsDir') + if self.weightsDir is not None: + # if we do have one specified, make sure it exists + if not os.path.exists(self.weightsDir): + err_handler.err_out_screen('ESMF Weights file directory specifed ({}) but does not exist').format( + self.weightsDir) + + # Calculate the beginning/ending processing dates if we are running realtime + if self.realtime_flag: + calculate_lookback_window(self) + + # Read in temporal interpolation options. + try: + self.forceTemoralInterp = json.loads(config['Interpolation']['ForcingTemporalInterpolation']) + except KeyError: + err_handler.err_out_screen('Unable to locate ForcingTemporalInterpolation under the Interpolation ' + 'section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ForcingTemporalInterpolation under the Interpolation ' + 'section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper ForcingTemporalInterpolation options specified in the ' + 'configuration file.') + if len(self.forceTemoralInterp) != self.number_inputs: + err_handler.err_out_screen('Please specify ForcingTemporalInterpolation values for each ' + 'corresponding input forcings in the configuration file.') + # Ensure the forcingTemporalInterpolation values make sense. + for temporalInterpOpt in self.forceTemoralInterp: + if temporalInterpOpt < 0 or temporalInterpOpt > 2: + err_handler.err_out_screen('Invalid ForcingTemporalInterpolation chosen in the configuration file. ' + 'Please choose a value of 0-2 for each corresponding input forcing.') + + # Read in the temperature downscaling options. + # Create temporary array to hold flags of if we need input parameter files. + param_flag = np.empty([len(self.input_forcings)], np.int) + param_flag[:] = 0 + try: + self.t2dDownscaleOpt = json.loads(config['Downscaling']['TemperatureDownscaling']) + except KeyError: + err_handler.err_out_screen('Unable to locate TemperatureDownscaling under the Downscaling ' + 'section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate TemperatureDownscaling under the Downscaling ' + 'section of the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper TemperatureDownscaling options specified in the configuration file.') + if len(self.t2dDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify TemperatureDownscaling values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + count_tmp = 0 + for optTmp in self.t2dDownscaleOpt: + if optTmp < 0 or optTmp > 2: + err_handler.err_out_screen('Invalid TemperatureDownscaling options specified in ' + 'the configuration file.') + if optTmp == 2: + param_flag[count_tmp] = 1 + count_tmp = count_tmp + 1 + + # Read in the pressure downscaling options. + try: + self.psfcDownscaleOpt = json.loads(config['Downscaling']['PressureDownscaling']) + except KeyError: + err_handler.err_out_screen('Unable to locate PressureDownscaling under the Downscaling ' + 'section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate PressureDownscaling under the Downscaling ' + 'section of the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper PressureDownscaling options specified in the configuration file.') + if len(self.psfcDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify PressureDownscaling values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + for optTmp in self.psfcDownscaleOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid PressureDownscaling options specified in the configuration file.') + + # Read in the shortwave downscaling options + try: + self.swDownscaleOpt = json.loads(config['Downscaling']['ShortwaveDownscaling']) + except KeyError: + err_handler.err_out_screen('Unable to locate ShortwaveDownscaling under the Downscaling ' + 'section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate ShortwaveDownscaling under the Downscaling ' + 'section of the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper ShortwaveDownscaling options specified in the configuration file.') + if len(self.swDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify ShortwaveDownscaling values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + for optTmp in self.swDownscaleOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid ShortwaveDownscaling options specified in the configuration file.') + + # Read in the precipitation downscaling options + try: + self.precipDownscaleOpt = json.loads(config['Downscaling']['PrecipDownscaling']) + except KeyError: + err_handler.err_out_screen('Unable to locate PrecipDownscaling under the Downscaling ' + 'section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate PrecipDownscaling under the Downscaling ' + 'section of the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper PrecipDownscaling options specified in the configuration file.') + if len(self.precipDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify PrecipDownscaling values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + count_tmp = 0 + for optTmp in self.precipDownscaleOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid PrecipDownscaling options specified in the configuration file.') + if optTmp == 1: + param_flag[count_tmp] = 1 + count_tmp = count_tmp + 1 + + # Read in humidity downscaling options. + try: + self.q2dDownscaleOpt = json.loads(config['Downscaling']['HumidityDownscaling']) + except KeyError: + err_handler.err_out_screen('Unable to locate HumidityDownscaling under the Downscaling ' + 'section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate HumidityDownscaling under the Downscaling ' + 'section of the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper HumidityDownscaling options specified in the configuration file.') + if len(self.q2dDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify HumidityDownscaling values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the downscaling options chosen make sense. + for optTmp in self.q2dDownscaleOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid HumidityDownscaling options specified in the configuration file.') + + # Read in the downscaling parameter directory. + self.paramFlagArray = param_flag + tmp_scale_param_dirs = [] + if param_flag.sum() > 0: + self.paramFlagArray = param_flag + try: + tmp_scale_param_dirs = config.get('Downscaling', 'DownscalingParamDirs').split(',') + except KeyError: + err_handler.err_out_screen('Unable to locate DownscalingParamDirs in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate DownscalingParamDirs in the configuration file.') + if len(tmp_scale_param_dirs) < param_flag.sum(): + err_handler.err_out_screen('Please specify a downscaling parameter directory for each ' + 'corresponding downscaling option that requires one.') + # Loop through each downscaling parameter directory and make sure they exist. + for dirTmp in range(0, len(tmp_scale_param_dirs)): + tmp_scale_param_dirs[dirTmp] = tmp_scale_param_dirs[dirTmp].strip() + if not os.path.isdir(tmp_scale_param_dirs[dirTmp]): + err_handler.err_out_screen('Unable to locate parameter directory: ' + tmp_scale_param_dirs[dirTmp]) + + # Create a list of downscaling parameter directories for each corresponding + # input forcing. If no directory is needed, or specified, we will set the value to NONE + self.dScaleParamDirs = [] + for count_tmp, _ in enumerate(self.input_forcings): + if param_flag[count_tmp] == 0: + self.dScaleParamDirs.append('NONE') + if param_flag[count_tmp] == 1: + self.dScaleParamDirs.append(tmp_scale_param_dirs[count_tmp]) + + # if the directory was specified but not downscaling, set it anyway for bias correction etc. + try: + if param_flag.sum() == 0 and len(config.get('Downscaling', 'DownscalingParamDirs').split(',')) == 1: + self.dScaleParamDirs = [config.get('Downscaling', 'DownscalingParamDirs').split(',')[0]] + except KeyError: + pass # TODO: this should not be `pass` if we have a parameter-based Bias Correction scheme selected + + # * Bias Correction Options * + + # Read AnA flag option + try: + # check both the Forecast section and if it's not there, the old BiasCorrection location + self.ana_flag = config['Forecast'].get('AnAFlag', config['BiasCorrection'].get('AnAFlag')) + if self.ana_flag is None: + raise KeyError + else: + self.ana_flag = int(self.ana_flag) + except KeyError: + err_handler.err_out_screen('Unable to locate AnAFlag in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate AnAFlag in the configuration file.') + except ValueError: + err_handler.err_out_screen('Improper AnAFlag value ') + if self.ana_flag < 0 or self.ana_flag > 1: + err_handler.err_out_screen('Please choose a AnAFlag value of 0 or 1.') + + # Read in temperature bias correction options + try: + self.t2BiasCorrectOpt = json.loads(config['BiasCorrection']['TemperatureBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate TemperatureBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate TemperatureBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper TemperatureBiasCorrection options specified in ' + 'the configuration file.') + if len(self.t2BiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify TemperatureBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.t2BiasCorrectOpt: + if optTmp < 0 or optTmp > 4: + err_handler.err_out_screen('Invalid TemperatureBiasCorrection options specified in the ' + 'configuration file.') + + # Read in surface pressure bias correction options. + try: + self.psfcBiasCorrectOpt = json.loads(config['BiasCorrection']['PressureBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate PressureBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate PressureBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper PressureBiasCorrection options specified in the configuration file.') + if len(self.psfcDownscaleOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify PressureBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.psfcBiasCorrectOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid PressureBiasCorrection options specified in the ' + 'configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in humidity bias correction options. + try: + self.q2BiasCorrectOpt = json.loads(config['BiasCorrection']['HumidityBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate HumidityBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate HumidityBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper HumdityBiasCorrection options specified in the configuration file.') + if len(self.q2BiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify HumidityBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.q2BiasCorrectOpt: + if optTmp < 0 or optTmp > 2: + err_handler.err_out_screen('Invalid HumidityBiasCorrection options specified in the ' + 'configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in wind bias correction options. + try: + self.windBiasCorrect = json.loads(config['BiasCorrection']['WindBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate WindBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate WindBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper WindBiasCorrection options specified in the configuration file.') + if len(self.windBiasCorrect) != self.number_inputs: + err_handler.err_out_screen('Please specify WindBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.windBiasCorrect: + if optTmp < 0 or optTmp > 4: + err_handler.err_out_screen('Invalid WindBiasCorrection options specified in the configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in shortwave radiation bias correction options. + try: + self.swBiasCorrectOpt = json.loads(config['BiasCorrection']['SwBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate SwBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SwBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper SwBiasCorrection options specified in the configuration file.') + if len(self.swBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify SwBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.swBiasCorrectOpt: + if optTmp < 0 or optTmp > 2: + err_handler.err_out_screen('Invalid SwBiasCorrection options specified in the configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in longwave radiation bias correction options. + try: + self.lwBiasCorrectOpt = json.loads(config['BiasCorrection']['LwBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate LwBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate LwBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper LwBiasCorrection options specified in ' + 'the configuration file.') + if len(self.lwBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify LwBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.lwBiasCorrectOpt: + if optTmp < 0 or optTmp > 4: + err_handler.err_out_screen('Invalid LwBiasCorrection options specified in the configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Read in precipitation bias correction options. + try: + self.precipBiasCorrectOpt = json.loads(config['BiasCorrection']['PrecipBiasCorrection']) + except KeyError: + err_handler.err_out_screen('Unable to locate PrecipBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate PrecipBiasCorrection under the ' + 'BiasCorrection section of the configuration file.') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper PrecipBiasCorrection options specified in the configuration file.') + if len(self.precipBiasCorrectOpt) != self.number_inputs: + err_handler.err_out_screen('Please specify PrecipBiasCorrection values for each corresponding ' + 'input forcings in the configuration file.') + # Ensure the bias correction options chosen make sense. + for optTmp in self.precipBiasCorrectOpt: + if optTmp < 0 or optTmp > 1: + err_handler.err_out_screen('Invalid PrecipBiasCorrection options specified in the configuration file.') + if optTmp == 1: + # We are running NWM-Specific bias-correction of CFSv2 that needs to take place prior to regridding. + self.runCfsNldasBiasCorrect = True + + # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on + # for ALL variables. + if self.runCfsNldasBiasCorrect: + if min(self.precipBiasCorrectOpt) != 1 and max(self.precipBiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'Precipitation under this configuration.') + if min(self.lwBiasCorrectOpt) != 1 and max(self.lwBiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'long-wave radiation under this configuration.') + if min(self.swBiasCorrectOpt) != 1 and max(self.swBiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'short-wave radiation under this configuration.') + if min(self.t2BiasCorrectOpt) != 1 and max(self.t2BiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'surface temperature under this configuration.') + if min(self.windBiasCorrect) != 1 and max(self.windBiasCorrect) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'wind forcings under this configuration.') + if min(self.q2BiasCorrectOpt) != 1 and max(self.q2BiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'specific humidity under this configuration.') + if min(self.psfcBiasCorrectOpt) != 1 and max(self.psfcBiasCorrectOpt) != 1: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for ' + 'surface pressure under this configuration.') + # Make sure we don't have any other forcings activated. This can only be ran for CFSv2. + for optTmp in self.input_forcings: + if optTmp != 7: + err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction can only be used in ' + 'CFSv2-only configurations') + + # Read in supplemental precipitation options as an array of values to map. + try: + self.supp_precip_forcings = json.loads(config['SuppForcing']['SuppPcp']) + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcp under SuppForcing section in configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcp under SuppForcing section in configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper SuppPcp option specified in configuration file') + self.number_supp_pcp = len(self.supp_precip_forcings) + + # Read in the supp pcp types (GRIB[1|2], NETCDF) + try: + self.supp_precip_file_types = config.get('SuppForcing', 'SuppPcpForcingTypes').strip("[]").split(',') + self.supp_precip_file_types = [stype.strip() for stype in self.supp_precip_file_types] + if self.supp_precip_file_types == ['']: + self.supp_precip_file_types = [] + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpForcingTypes in SuppForcing section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpForcingTypes in SuppForcing section ' + 'in the configuration file.') + if len(self.supp_precip_file_types) != self.number_supp_pcp: + err_handler.err_out_screen('Number of SuppPcpForcingTypes ({}) must match the number ' + 'of SuppPcp inputs ({}) in the configuration file.'.format(len(self.supp_precip_file_types), self.number_supp_pcp)) + for fileType in self.supp_precip_file_types: + if fileType not in ['GRIB1', 'GRIB2', 'NETCDF']: + err_handler.err_out_screen('Invalid SuppForcing file type "{}" specified. ' + 'Only GRIB1, GRIB2, and NETCDF are supported'.format(fileType)) + + if self.number_supp_pcp > 0: + # Check to make sure supplemental precip options make sense. Also read in the RQI threshold + # if any radar products where chosen. + for suppOpt in self.supp_precip_forcings: + if suppOpt < 0 or suppOpt > 11: + err_handler.err_out_screen('Please specify SuppForcing values between 1 and 7.') + # Read in RQI threshold to apply to radar products. + if suppOpt == 1 or suppOpt == 2: + try: + self.rqiMethod = json.loads(config['SuppForcing']['RqiMethod']) + except KeyError: + err_handler.err_out_screen('Unable to locate RqiMethod under SuppForcing ' + 'section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RqiMethod under SuppForcing ' + 'section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper RqiMethod option in the configuration file.') + # Make sure the RqiMethod makes sense. + if self.rqiMethod < 0 or self.rqiMethod > 2: + err_handler.err_out_screen('Please specify an RqiMethod of either 0, 1, or 2.') + + try: + self.rqiThresh = json.loads(config['SuppForcing']['RqiThreshold']) + except KeyError: + err_handler.err_out_screen('Unable to locate RqiThreshold under ' + 'SuppForcing section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RqiThreshold under ' + 'SuppForcing section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper RqiThreshold option in the configuration file.') + # Make sure the RQI threshold makes sense. + if self.rqiThresh < 0.0 or self.rqiThresh > 1.0: + err_handler.err_out_screen('Please specify an RqiThreshold between 0.0 and 1.0.') + + # Read in the input directories for each supplemental precipitation product. + try: + self.supp_precip_dirs = config.get('SuppForcing', 'SuppPcpDirectories').split(',') + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section ' + 'in the configuration file.') + if len(self.supp_precip_dirs) != self.number_supp_pcp: + err_handler.err_out_screen('Number of SuppPcpDirectories must match the number ' + 'of SuppForcing in the configuration file.') + # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace + # or new line characters. + for dirTmp in range(0, len(self.supp_precip_dirs)): + self.supp_precip_dirs[dirTmp] = self.supp_precip_dirs[dirTmp].strip() + if not os.path.isdir(self.supp_precip_dirs[dirTmp]): + err_handler.err_out_screen('Unable to locate supp pcp directory: ' + self.supp_precip_dirs[dirTmp]) + + # Process supplemental precipitation enforcement options + try: + self.supp_precip_mandatory = json.loads(config['SuppForcing']['SuppPcpMandatory']) + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpMandatory under the SuppForcing section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpMandatory under the SuppForcing section ' + 'in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper SuppPcpMandatory options specified in the configuration file.') + if len(self.supp_precip_mandatory) != self.number_supp_pcp: + err_handler.err_out_screen('Please specify SuppPcpMandatory values for each corresponding ' + 'supplemental precipitation options in the configuration file.') + # Check to make sure enforcement options makes sense. + for enforceOpt in self.supp_precip_mandatory: + if enforceOpt < 0 or enforceOpt > 1: + err_handler.err_out_screen('Invalid SuppPcpMandatory chosen in the configuration file. ' + 'Please choose a value of 0 or 1 for each corresponding ' + 'supplemental precipitation product.') + + # Read in the regridding options. + try: + self.regrid_opt_supp_pcp = json.loads(config['SuppForcing']['RegridOptSuppPcp']) + except KeyError: + err_handler.err_out_screen('Unable to locate RegridOptSuppPcp under the SuppForcing section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate RegridOptSuppPcp under the SuppForcing section ' + 'in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper RegridOptSuppPcp options specified in the configuration file.') + if len(self.regrid_opt_supp_pcp) != self.number_supp_pcp: + err_handler.err_out_screen('Please specify RegridOptSuppPcp values for each corresponding supplemental ' + 'precipitation product in the configuration file.') + # Check to make sure regridding options makes sense. + for regridOpt in self.regrid_opt_supp_pcp: + if regridOpt < 1 or regridOpt > 3: + err_handler.err_out_screen('Invalid RegridOptSuppPcp chosen in the configuration file. ' + 'Please choose a value of 1-3 for each corresponding ' + 'supplemental precipitation product.') + + # Read in temporal interpolation options. + try: + self.suppTemporalInterp = json.loads(config['SuppForcing']['SuppPcpTemporalInterpolation']) + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpTemporalInterpolation under the SuppForcing ' + 'section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpTemporalInterpolation under the SuppForcing ' + 'section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper SuppPcpTemporalInterpolation options specified in the ' + 'configuration file.') + if len(self.suppTemporalInterp) != self.number_supp_pcp: + err_handler.err_out_screen('Please specify SuppPcpTemporalInterpolation values for each ' + 'corresponding supplemental precip products in the configuration file.') + # Ensure the SuppPcpTemporalInterpolation values make sense. + for temporalInterpOpt in self.suppTemporalInterp: + if temporalInterpOpt < 0 or temporalInterpOpt > 2: + err_handler.err_out_screen('Invalid SuppPcpTemporalInterpolation chosen in the configuration file. ' + 'Please choose a value of 0-2 for each corresponding input forcing') + + # Read in the SuppPcpInputOffsets options. + try: + self.supp_input_offsets = json.loads(config['SuppForcing']['SuppPcpInputOffsets']) + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpInputOffsets under SuppForcing ' + 'section in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpInputOffsets under SuppForcing ' + 'section in the configuration file.') + except json.decoder.JSONDecodeError: + err_handler.err_out_screen('Improper SuppPcpInputOffsets option specified in ' + 'the configuration file.') + if len(self.supp_input_offsets) != self.number_supp_pcp: + err_handler.err_out_screen('Please specify SuppPcpInputOffsets values for each ' + 'corresponding input forcings for SuppForcing.') + # Check to make sure the input offset options make sense. There will be additional + # checking later when input choices are mapped to input products. + for inputOffset in self.supp_input_offsets: + if inputOffset < 0: + err_handler.err_out_screen( + 'Please specify SuppPcpInputOffsets values greater than or equal to zero.') + + # Read in the optional parameter directory for supplemental precipitation. + try: + self.supp_precip_param_dir = config['SuppForcing']['SuppPcpParamDir'] + except KeyError: + err_handler.err_out_screen('Unable to locate SuppPcpParamDir under the SuppForcing section ' + 'in the configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate SuppPcpParamDir under the SuppForcing section ' + 'in the configuration file.') + except ValueError: + err_handler.err_out_screen('Improper SuppPcpParamDir option specified in the configuration file.') + if not os.path.isdir(self.supp_precip_param_dir): + err_handler.err_out_screen('Unable to locate SuppPcpParamDir: ' + self.supp_precip_param_dir) + + # Read in Ensemble information + # Read in CFS ensemble member information IF we have chosen CFSv2 as an input + # forcing. + for optTmp in self.input_forcings: + if optTmp == 7: + try: + self.cfsv2EnsMember = json.loads(config['Ensembles']['cfsEnsNumber']) + except KeyError: + err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles ' + 'section of the configuration file') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles ' + 'section of the configuration file') + except json.JSONDecodeError: + err_handler.err_out_screen('Improper cfsEnsNumber options specified in the configuration file') + if self.cfsv2EnsMember < 1 or self.cfsv2EnsMember > 4: + err_handler.err_out_screen('Please chose an cfsEnsNumber value of 1,2,3 or 4.') + + # Read in information for the custom input NetCDF files that are to be processed. + # Read in the ForecastInputHorizons options. + try: + self.customFcstFreq = json.loads(config['Custom']['custom_input_fcst_freq']) + except KeyError: + err_handler.err_out_screen('Unable to locate custom_input_fcst_freq under Custom section in ' + 'configuration file.') + except configparser.NoOptionError: + err_handler.err_out_screen('Unable to locate custom_input_fcst_freq under Custom section in ' + 'configuration file.') + except json.decoder.JSONDecodeError as je: + err_handler.err_out_screen('Improper custom_input_fcst_freq option specified in ' + 'configuration file: ' + str(je)) + if len(self.customFcstFreq) != self.number_custom_inputs: + err_handler.err_out_screen(f'Improper custom_input fcst_freq specified. ' + f'This number ({len(self.customFcstFreq)}) must ' + f'match the frequency of custom input forcings selected ' + f'({self.number_custom_inputs}).') + +def calculate_lookback_window(config_options): + """ + Calculate the beginning, ending datetime variables + for a look-back period. Also calculate the processing + time window delta value. + :param config_options: Abstract class holding job information. + :return: Updated abstract class with updated datetime variables. + """ + # First calculate the current time in UTC. + if config_options.realtime_flag: + d_current_utc = datetime.datetime.utcnow() + else: + d_current_utc = config_options.b_date_proc + + # Next, subtract the lookup window (specified in minutes) to get a crude window + # of processing. + d_lookback = d_current_utc - datetime.timedelta( + seconds=60 * (config_options.look_back - config_options.output_freq)) + + # Determine the first forecast iteration that will be processed on this day + # based on the forecast frequency and where in the day we are at. + fcst_step_tmp = math.ceil((d_lookback.hour * 60 + d_lookback.minute) / config_options.fcst_freq) + d_look_tmp1 = datetime.datetime(d_lookback.year, d_lookback.month, d_lookback.day) + d_look_tmp1 = d_look_tmp1 + datetime.timedelta(seconds=(60 * config_options.fcst_freq * fcst_step_tmp)) + + # If we are offsetting the forecasts, apply here. + if config_options.fcst_shift > 0: + d_look_tmp1 = d_look_tmp1 + datetime.timedelta(seconds=60 * config_options.fcst_shift) + + config_options.b_date_proc = d_look_tmp1 + + # Now calculate the end of the processing window based on the time from the + # beginning of the processing window. + dt_tmp = d_current_utc - config_options.b_date_proc + n_fcst_steps = math.floor((dt_tmp.days*1440+dt_tmp.seconds/60.0) / config_options.fcst_freq) + config_options.nFcsts = int(n_fcst_steps) + 1 + config_options.e_date_proc = config_options.b_date_proc + datetime.timedelta( + seconds=n_fcst_steps * config_options.fcst_freq * 60) +def main(): + if len(sys.argv) < 2: + print("%s config" % sys.argv[0]) + sys.exit(1) + + config_file = sys.argv[1] + + config = ConfigOptions(config_file) + config.read_config() + from pprint import pprint + pprint(vars(config)) + + +if __name__ == '__main__': + main() diff --git a/core/disaggregateMod.py b/core/disaggregateMod.py index 9c50756..ad4281d 100755 --- a/core/disaggregateMod.py +++ b/core/disaggregateMod.py @@ -7,13 +7,14 @@ import numpy as np from netCDF4 import Dataset - +from strenum import StrEnum from core import err_handler +from core.enumConfig import SuppForcingPcpEnum test_enabled = True def disaggregate_factory(ConfigOptions): - if len(ConfigOptions.supp_precip_forcings) == 1 and ConfigOptions.supp_precip_forcings[0] == 11: + if len(ConfigOptions.supp_precip_forcings) == 1 and ConfigOptions.supp_precip_forcings[0] == str(SuppForcingPcpEnum.AK_NWS_IV): return ext_ana_disaggregate #Add new cases here #elif condition: @@ -113,9 +114,9 @@ def ext_ana_disaggregate(input_forcings, supplemental_precip, config_options, mp ana_sum = np.array([],dtype=np.float32) target_data = np.array([],dtype=np.float32) - ana_all_zeros = np.array([],dtype=np.bool) - ana_no_zeros = np.array([],dtype=np.bool) - target_data_no_zeros = np.array([],dtype=np.bool) + ana_all_zeros = np.array([],dtype=bool) + ana_no_zeros = np.array([],dtype=bool) + target_data_no_zeros = np.array([],dtype=bool) if mpi_config.rank == 0: config_options.statusMsg = f"Performing hourly disaggregation of {supplemental_precip.file_in2}" err_handler.log_msg(config_options, mpi_config) @@ -151,9 +152,6 @@ def ext_ana_disaggregate(input_forcings, supplemental_precip, config_options, mp 0) np.seterr(**orig_err_settings) - disagg_factors = np.select([ana_all_zeros,(ana_no_zeros | target_data_no_zeros)], - [1/6.0*np.ones(supplemental_precip.regridded_precip2[:,:].shape),np.clip(target_data/ana_sum,0,1)], - 0) if mpi_config.comm.Get_size() == 1 and test_enabled: test_file = f"{config_options.scratch_dir}/stage_4_acc6h_{yyyymmdd}_{beg_hh}_{end_hh}.txt" np.savetxt(test_file,supplemental_precip.regridded_precip2) diff --git a/core/downscale.py b/core/downscale.py index 281ab11..ad6abb4 100755 --- a/core/downscale.py +++ b/core/downscale.py @@ -6,13 +6,17 @@ import math import os import time - +from strenum import StrEnum import numpy as np from netCDF4 import Dataset - +import yaml +from core.enumConfig import DownScaleTempEnum +from core.enumConfig import DownScalePressEnum +from core.enumConfig import DownScaleSwEnum +from core.enumConfig import DownScalePrecipEnum +from core.enumConfig import DownScaleHumidEnum from core import err_handler - def run_downscaling(input_forcings, config_options, geo_meta_wrf_hydro, mpi_config): """ Top level module function that will downscale forcing variables @@ -25,9 +29,9 @@ def run_downscaling(input_forcings, config_options, geo_meta_wrf_hydro, mpi_conf """ # Dictionary mapping to temperature downscaling. downscale_temperature = { - 0: no_downscale, - 1: simple_lapse, - 2: param_lapse + str(DownScaleTempEnum.NONE.name) : no_downscale, + str(DownScaleTempEnum.LAPSE_675.name) : simple_lapse, + str(DownScaleTempEnum.LAPSE_PRE_CALC.name): param_lapse } downscale_temperature[input_forcings.t2dDownscaleOpt](input_forcings, config_options, geo_meta_wrf_hydro, mpi_config) @@ -35,8 +39,8 @@ def run_downscaling(input_forcings, config_options, geo_meta_wrf_hydro, mpi_conf # Dictionary mapping to pressure downscaling. downscale_pressure = { - 0: no_downscale, - 1: pressure_down_classic + str(DownScalePressEnum.NONE.name): no_downscale, + str(DownScalePressEnum.ELEV.name): pressure_down_classic } downscale_pressure[input_forcings.psfcDownscaleOpt](input_forcings, config_options, geo_meta_wrf_hydro, mpi_config) @@ -44,33 +48,30 @@ def run_downscaling(input_forcings, config_options, geo_meta_wrf_hydro, mpi_conf # Dictionary mapping to shortwave radiation downscaling downscale_sw = { - 0: no_downscale, - 1: ncar_topo_adj + str(DownScaleSwEnum.NONE.name): no_downscale, + str(DownScaleSwEnum.ELEV.name): ncar_topo_adj } downscale_sw[input_forcings.swDowscaleOpt](input_forcings, config_options, geo_meta_wrf_hydro, mpi_config) err_handler.check_program_status(config_options, mpi_config) # Dictionary mapping to specific humidity downscaling downscale_q2 = { - 0: no_downscale, - 1: q2_down_classic + str(DownScaleHumidEnum.NONE.name) : no_downscale, + str(DownScaleHumidEnum.REGRID_TEMP_PRESS.name): q2_down_classic } downscale_q2[input_forcings.q2dDownscaleOpt](input_forcings, config_options, geo_meta_wrf_hydro, mpi_config) err_handler.check_program_status(config_options, mpi_config) # Dictionary mapping to precipitation downscaling. downscale_precip = { - 0: no_downscale, - 1: nwm_monthly_PRISM_downscale + str(DownScalePrecipEnum.NONE.name) : no_downscale, + str(DownScalePrecipEnum.NWM_MM.name): nwm_monthly_PRISM_downscale #1: precip_mtn_mapper } downscale_precip[input_forcings.precipDownscaleOpt](input_forcings, config_options, geo_meta_wrf_hydro, mpi_config) err_handler.check_program_status(config_options, mpi_config) - - - def no_downscale(input_forcings, ConfigOptions, GeoMetaWrfHydro, MpiConfig): """ Generic function for passing states through without any @@ -104,10 +105,14 @@ def simple_lapse(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): elevDiff = input_forcings.height - GeoMetaWrfHydro.height + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) # Assign existing, un-downscaled temperatures to a temporary placeholder, which # will be used for specific humidity downscaling. - if input_forcings.q2dDownscaleOpt > 0: - input_forcings.t2dTmp[:,:] = input_forcings.final_forcings[4,:,:] + if input_forcings.q2dDownscaleOpt != DownScaleHumidEnum.NONE.name: + ind = int(outConfig['T2D']['ind']) + input_forcings.t2dTmp[:,:] = input_forcings.final_forcings[ind,:,:] # Apply single lapse rate value to the input 2-meter # temperature values. @@ -118,7 +123,8 @@ def simple_lapse(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): err_handler.log_critical(ConfigOptions, MpiConfig) return try: - input_forcings.final_forcings[4,:,:] = input_forcings.final_forcings[4,:,:] + \ + ind = int(outConfig['T2D']['ind']) + input_forcings.final_forcings[ind,:,:] = input_forcings.final_forcings[ind,:,:] + \ (6.49/1000.0)*elevDiff except: ConfigOptions.errMsg = "Unable to apply lapse rate to input 2-meter temperatures." @@ -239,9 +245,13 @@ def param_lapse(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): # Scatter the lapse rate grid to the other processors. input_forcings.lapseGrid = MpiConfig.scatter_array(GeoMetaWrfHydro,lapseTmp,ConfigOptions) err_handler.check_program_status(ConfigOptions, MpiConfig) - + + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) # Apply the local lapse rate grid to our local slab of 2-meter temperature data. - temperature_grid_tmp = input_forcings.final_forcings[4, :, :] + ind = int(outConfig['T2D']['ind']) + temperature_grid_tmp = input_forcings.final_forcings[ind, :, :] try: indNdv = np.where(input_forcings.final_forcings == ConfigOptions.globalNdv) except: @@ -265,7 +275,7 @@ def param_lapse(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): err_handler.log_critical(ConfigOptions, MpiConfig) return - input_forcings.final_forcings[4,:,:] = temperature_grid_tmp + input_forcings.final_forcings[ind,:,:] = temperature_grid_tmp input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv # Reset for memory efficiency @@ -294,10 +304,14 @@ def pressure_down_classic(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig return elevDiff = input_forcings.height - GeoMetaWrfHydro.height + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) # Assign existing, un-downscaled pressure values to a temporary placeholder, which # will be used for specific humidity downscaling. - if input_forcings.q2dDownscaleOpt > 0: - input_forcings.psfcTmp[:, :] = input_forcings.final_forcings[6, :, :] + ind = int(outConfig['PSFC']['ind']) + if input_forcings.q2dDownscaleOpt != DownScaleHumidEnum.NONE.name: + input_forcings.psfcTmp[:, :] = input_forcings.final_forcings[ind, :, :] try: indNdv = np.where(input_forcings.final_forcings == ConfigOptions.globalNdv) @@ -306,7 +320,7 @@ def pressure_down_classic(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig err_handler.log_critical(ConfigOptions, MpiConfig) return try: - input_forcings.final_forcings[6,:,:] = input_forcings.final_forcings[6,:,:] +\ + input_forcings.final_forcings[ind,:,:] = input_forcings.final_forcings[ind,:,:] +\ (input_forcings.final_forcings[6,:,:]*elevDiff*9.8)/\ (input_forcings.final_forcings[4,:,:]*287.05) except: @@ -359,7 +373,12 @@ def q2_down_classic(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): "incoming specific humidity" err_handler.log_critical(ConfigOptions, MpiConfig) return - input_forcings.final_forcings[5,:,:] = q2Tmp + + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) + ind = int(outConfig['Q2D']['ind']) + input_forcings.final_forcings[ind,:,:] = q2Tmp input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv q2Tmp = None indNdv = None @@ -527,7 +546,11 @@ def nwm_monthly_PRISM_downscale(input_forcings,ConfigOptions,GeoMetaWrfHydro,Mpi input_forcings.nwmPRISM_denGrid = MpiConfig.scatter_array(GeoMetaWrfHydro, denDataTmp, ConfigOptions) err_handler.check_program_status(ConfigOptions, MpiConfig) + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) # Create temporary grids from the local slabs of params/precip forcings. + ind = int(outConfig['RAINRATE']['ind']) localRainRate = input_forcings.final_forcings[3,:,:] numLocal = input_forcings.nwmPRISM_numGrid[:,:] denLocal = input_forcings.nwmPRISM_denGrid[:,:] @@ -571,7 +594,7 @@ def nwm_monthly_PRISM_downscale(input_forcings,ConfigOptions,GeoMetaWrfHydro,Mpi err_handler.log_critical(ConfigOptions, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - input_forcings.final_forcings[3, :, :] = localRainRate + input_forcings.final_forcings[ind, :, :] = localRainRate # Reset variables for memory efficiency idDenom = None @@ -621,7 +644,7 @@ def ncar_topo_adj(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig): return try: - TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,coszen_loc,DECLIN,SOLCON, + TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,ConfigOptions,coszen_loc,DECLIN,SOLCON, hrang_loc) except: ConfigOptions.errMsg = "Unable to perform final topographic adjustment of incoming " \ @@ -711,7 +734,7 @@ def calc_coszen(ConfigOptions,declin,GeoMetaWrfHydro): return coszen, hrang -def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,COSZEN,declin,solcon,hrang2d): +def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,ConfigOptions,COSZEN,declin,solcon,hrang2d): """ Downscaling driver for correcting incoming shortwave radiation fluxes from a low resolution to a a higher resolution. @@ -731,21 +754,27 @@ def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,COSZEN,declin,solcon,hrang2 xxlat = GeoMetaWrfHydro.latitude_grid*degrad # Sanity checking on incoming shortwave grid. - SWDOWN = input_forcings.final_forcings[7,:,:] + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) + ind = int(outConfig['SWDOWN']['ind']) + SWDOWN = input_forcings.final_forcings[ind,:,:] SWDOWN[np.where(SWDOWN < 0.0)] = 0.0 SWDOWN[np.where(SWDOWN >= 1400.0)] = 1400.0 COSZEN[np.where(COSZEN < 1E-4)] = 1E-4 - corr_frac = np.empty([ny, nx], np.int) - # shadow_mask = np.empty([ny,nx],np.int) - diffuse_frac = np.empty([ny, nx], np.int) + corr_frac = np.empty([ny, nx], int) + + # shadow_mask = np.empty([ny,nx],int) + diffuse_frac = np.empty([ny, nx], int) corr_frac[:, :] = 0 diffuse_frac[:, :] = 0 # shadow_mask[:,:] = 0 indTmp = np.where((GeoMetaWrfHydro.slope[:,:] == 0.0) & (SWDOWN <= 10.0)) + corr_frac[indTmp] = 1 term1 = np.sin(xxlat) * np.cos(hrang2d) @@ -783,7 +812,7 @@ def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,COSZEN,declin,solcon,hrang2 term5 = None term6 = None - input_forcings.final_forcings[7,:,:] = SWDOWN_OUT + input_forcings.final_forcings[ind,:,:] = SWDOWN_OUT # Reset variables to free up memory SWDOWN = None @@ -798,7 +827,11 @@ def rel_hum(input_forcings,ConfigOptions): :param ConfigOptions: :return: """ - tmpHumidity = input_forcings.final_forcings[5,:,:]/(1-input_forcings.final_forcings[5,:,:]) + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) + ind = int(outConfig['Q2D']['ind']) + tmpHumidity = input_forcings.final_forcings[ind,:,:]/(1-input_forcings.final_forcings[ind,:,:]) T0 = 273.15 EP = 0.622 @@ -830,12 +863,16 @@ def mixhum_ptrh(input_forcings,relHum,iswit,ConfigOptions): ES0 = 6.11 A = 17.269 B = 35.86 - - term1 = A * (input_forcings.final_forcings[4,:,:] - T0) - term2 = input_forcings.final_forcings[4,:,:] - B + outputVarAttrYaml = ConfigOptions.outputVarAttrYaml + out_yaml_stream = open(outputVarAttrYaml) + outConfig = yaml.safe_load(out_yaml_stream) + ind1 = int(outConfig['T2D']['ind']) + term1 = A * (input_forcings.final_forcings[ind1,:,:] - T0) + term2 = input_forcings.final_forcings[ind1,:,:] - B EST = np.exp(term1 / term2) * ES0 - QST = (EP * EST) / ((input_forcings.final_forcings[6,:,:]/100.0) - ONEMEP * EST) + ind2 = int(outConfig['PSFC']['ind']) + QST = (EP * EST) / ((input_forcings.final_forcings[ind2,:,:]/100.0) - ONEMEP * EST) QW = QST * (relHum * 0.01) if iswit == 2: QW = QW / (1.0 + QW) diff --git a/core/enumConfig.py b/core/enumConfig.py new file mode 100644 index 0000000..f3d2935 --- /dev/null +++ b/core/enumConfig.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +import os +import sys +import enum +from strenum import StrEnum +from enum import IntEnum + +class ForcingEnum(IntEnum): + NLDAS = 1 #GRIB retrospective files + NARR = 2 #GRIB retrospective files + GFS_GLOBAL = 3 #GRIB2 Global production files on the full gaussian grid + NAM_NEST_CONUS = 4 #Nest GRIB2 Conus production files + HRRR = 5 #GRIB2 Conus production files + RAP = 6 #GRIB2 Conus 13km production files + CFS_V2 = 7 #6-hourly GRIB2 Global production files + WRF_NEST_HI = 8 #GRIB2 Hawaii nest files + GFS_GLOBAL_25 = 9 #GRIB2 Global production files on 0.25 degree lat/lon grids. + CUSTOM_1 = 10 #Custom NetCDF hourly forcing files + CUSTOM_2 = 11 #NetCDF hourly forcing files + CUSTOM_3 = 12 #NetCDF hourly forcing files + NAM_NEST_HI = 13 #3-km NAM Nest. + NAM_NEST_PR = 14 #3-km NAM Nest. + NAM_NEST_AK = 15 #3-km Alaska Nest + NAM_NEST_HI_RAD = 16 #NAM_Nest_3km_Hawaii_Radiation-Only + NAM_NEST_PR_RAD = 17 #NAM_Nest_3km_PuertoRico_Radiation-Only + WRF_ARW_PR = 18 #GRIB2 PuertoRico + HRRR_AK = 19 #HRRR GRIB2 Alaska production files + HRRR_AK_EXT = 20 #ExtAna HRRR AK FE output + +class ForcingTypeEnum(StrEnum): + GRIB1 = "GRIB1" + GRIB2 = "GRIB2" + NETCDF = "NETCDF" + +class RegriddingOptEnum(StrEnum): + ESMF_BILINEAR = "ESMF_BILINEAR" + ESMF_NEAREST_NEIGHBOR = "ESMF_NEAREST_NEIGHBOR" + ESMF_CONSERVATIVE_BILINEAR = "ESMF_CONSERVATIVE_BILINEAR" + +class TemporalInterpEnum(StrEnum): + NONE = "NONE" + NEAREST_NEIGHBOR = "NEAREST_NEIGHBOR" + LINEAR_WEIGHT_AVG = "LINEAR_WEIGHT_AVG" + +class BiasCorrTempEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + CUSTOM = "CUSTOM" + GFS = "GFS" + HRRR = "HRRR" + +class BiasCorrPressEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + +class BiasCorrHumidEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + CUSTOM = "CUSTOM" + +class BiasCorrWindEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + CUSTOM = "CUSTOM" + GFS = "GFS" + HRRR = "HRRR" + +class BiasCorrSwEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + CUSTOM = "CUSTOM" + +class BiasCorrLwEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + CUSTOM = "CUSTOM" + GFS = "GFS" + +class BiasCorrPrecipEnum(StrEnum): + NONE = "NONE" + CFS_V2 = "CFS_V2" + +class DownScaleTempEnum(StrEnum): + NONE = "NONE" + LAPSE_675 = "LAPSE_675" + LAPSE_PRE_CALC = "LAPSE_PRE_CALC" + +class DownScalePressEnum(StrEnum): + NONE = "NONE" + ELEV = "ELEV" + +class DownScaleSwEnum(StrEnum): + NONE = "NONE" + ELEV = "ELEV" + +class DownScalePrecipEnum(StrEnum): + NONE = "NONE" + NWM_MM = "NWM_MM" + +class DownScaleHumidEnum(StrEnum): + NONE = "NONE" + REGRID_TEMP_PRESS = "REGRID_TEMP_PRESS" + +class OutputFloatEnum(StrEnum): + SCALE_OFFSET = "SCALE_OFFSET" + FLOAT = "FLOAT" + +class SuppForcingRqiMethodEnum(StrEnum): + NONE = "NONE" + MRMS = "MRMS" + NWM = "NWM" + +class SuppForcingPcpEnum(StrEnum): + MRMS = "MRMS" + MRMS_GAGE = "MRMS_GAGE" + WRF_ARW_HI = "WRF_ARW_HI" + WRF_ARW_PR = "WRF_ARW_PR" + MRMS_CONUS_MS = "MRMS_CONUS_MS" + MRMS_HI_MS = "MRMS_HI_MS" + MRMS_SBCV2 = "MRMS_SBCV2" + AK_OPT1 = "AK_OPT1" + AK_OPT2 = "AK_OPT2" + AK_MRMS = "AK_MRMS" + AK_NWS_IV = "AK_NWS_IV" + +class RegriddingOptEnum(IntEnum): + ESMF_BILINEAR = 1 + ESMF_NEAREST_NEIGHBOR = 2 + ESMF_CONSERVATIVE_BILINEAR = 3 + +class TemporalInterpEnum(IntEnum): + NONE = 0 + NEAREST_NEIGHBOR = 1 + LINEAR_WEIGHT_AVG = 2 + +class BiasCorrTempEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + CUSTOM = 2 + GFS = 3 + HRRR = 4 + +class BiasCorrPressEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + +class BiasCorrHumidEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + CUSTOM = 2 + +class BiasCorrWindEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + CUSTOM = 2 + GFS = 3 + HRRR = 4 + +class BiasCorrSwEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + CUSTOM = 2 + +class BiasCorrLwEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + CUSTOM = 2 + GFS = 3 + +class BiasCorrPrecipEnum(IntEnum): + NONE = 0 + CFS_V2 = 1 + +class DownScaleTempEnum(IntEnum): + NONE = 0 + LAPSE_675 = 1 + LAPSE_PRE_CALC = 2 + +class DownScalePressEnum(IntEnum): + NONE = 0 + ELEV = 1 + +class DownScaleSwEnum(IntEnum): + NONE = 0 + ELEV = 1 + +class DownScalePrecipEnum(IntEnum): + NONE = 0 + NWM_MM = 1 + +class DownScaleHumidEnum(IntEnum): + NONE = 0 + REGRID_TEMP_PRESS = 1 + +class OutputFloatEnum(IntEnum): + SCALE_OFFSET = 0 + FLOAT = 1 + +class SuppForcingPcpEnum(IntEnum): + MRMS = 1 + MRMS_GAGE = 2 + WRF_ARW_HI = 3 + WRF_ARW_PR = 4 + MRMS_CONUS_MS = 5 + MRMS_HI_MS = 6 + MRMS_SBCV2 = 7 + AK_OPT1 = 8 + AK_OPT2 = 9 + AK_MRMS = 10 + AK_NWS_IV = 11 + +class SuppForcingRqiMethodEnum(IntEnum): + NONE = 0 + MRMS = 1 + NWM = 2 diff --git a/core/err_handler.py b/core/err_handler.py index 47e7354..cf30818 100755 --- a/core/err_handler.py +++ b/core/err_handler.py @@ -259,18 +259,21 @@ def check_forcing_bounds(ConfigOptions, input_forcings, MpiConfig): :param MpiConfig: :return: """ + OutputEnum = ConfigOptions.OutputEnum # Establish a range of values for each output variable. variable_range = { - 'U2D': [0, -500.0, 500.0], - 'V2D': [1, -500.0, 500.0], - 'LWDOWN': [2, -1000.0, 10000.0], - 'RAINRATE': [3, 0.0, 100.0], - 'T2D': [4, 0.0, 400.0], - 'Q2D': [5, -100.0, 100.0], - 'PSFC': [6, 0.0, 2000000.0], - 'SWDOWN': [7, 0.0, 5000.0] + OutputEnum.U2D.name : [0, -500.0, 500.0], + OutputEnum.V2D.name : [1, -500.0, 500.0], + OutputEnum.LWDOWN.name : [2, -1000.0, 10000.0], + OutputEnum.RAINRATE.name: [3, 0.0, 100.0], + OutputEnum.T2D.name : [4, 0.0, 400.0], + OutputEnum.Q2D.name : [5, -100.0, 100.0], + OutputEnum.PSFC.name : [6, 0.0, 2000000.0], + OutputEnum.SWDOWN.name : [7, 0.0, 5000.0] + #OutputEnum.LQFRAC.name : [8, ,] } - fvars = ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', 'Q2D', 'PSFC', 'SWDOWN'] + #fvars = ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', 'Q2D', 'PSFC', 'SWDOWN'] + fvars = [str(item.name) for item in OutputEnum] # If the regridded field is None type, return to the main program as this means no forcings # were found for this timestep. @@ -280,7 +283,7 @@ def check_forcing_bounds(ConfigOptions, input_forcings, MpiConfig): # Loop over all the variables. Check for reasonable ranges. If any values are # exceeded, shut the forcing engine down. for varTmp in variable_range: - if fvars.index(varTmp) not in input_forcings.input_map_output: + if varTmp not in input_forcings.input_map_output: continue # First check to see if we have any data that is not missing. diff --git a/core/forcingInputMod.py b/core/forcingInputMod.py old mode 100755 new mode 100644 index c9d23d3..f20bac9 --- a/core/forcingInputMod.py +++ b/core/forcingInputMod.py @@ -5,71 +5,73 @@ initializing ESMF grids and regrid objects), etc """ import numpy as np - +from strenum import StrEnum from core import time_handling -from core import regrid -from core import timeInterpMod - +#from core import regrid +#from core import timeInterpMod +import yaml +from core.enumConfig import TemporalInterpEnum +from core.enumConfig import DownScaleHumidEnum class input_forcings: """ This is an abstract class that will define all the parameters of a single input forcing product. """ - + # Constants - GRIB2 = "GRIB2" - GRIB1 = "GRIB1" + GRIB2 = "GRIB2" + GRIB1 = "GRIB1" NETCDF = "NETCDF" - + def __init__(self): """ Initializing all attributes and objects to None. """ - self.keyValue = None - self.inDir = None - self.enforce = None - self.paramDir = None - self.userFcstHorizon = None - self.userCycleOffset = None - self.productName = None - self.fileType = None - self.file_ext = None - self.nx_global = None - self.ny_global = None - self.nx_local = None - self.ny_local = None - self.x_lower_bound = None - self.x_upper_bound = None - self.y_lower_bound = None - self.y_upper_bound = None - self.cycleFreq = None - self.outFreq = None - self.regridOpt = None - self.timeInterpOpt = None - self.t2dDownscaleOpt = None - self.lapseGrid = None - self.rqiClimoGrid = None - self.swDowscaleOpt = None - self.precipDownscaleOpt = None - self.nwmPRISM_numGrid = None - self.nwmPRISM_denGrid = None - self.q2dDownscaleOpt = None - self.psfcDownscaleOpt = None - self.t2dBiasCorrectOpt = None - self.swBiasCorrectOpt = None - self.precipBiasCorrectOpt = None - self.q2dBiasCorrectOpt = None - self.windBiasCorrectOpt = None - self.psfcBiasCorrectOpt = None - self.lwBiasCorrectOpt = None - self.esmf_lats = None - self.esmf_lons = None - self.esmf_grid_in = None - self.regridComplete = False - self.regridObj = None - self.esmf_field_in = None - self.esmf_field_out = None + self.keyValue = None + self.inDir = None + self.enforce = None + self.paramDir = None + self.userFcstHorizon = None + self.userCycleOffset = None + self.productName = None + self.fileType = None + self.file_ext = None + self.nx_global = None + self.ny_global = None + self.nx_local = None + self.ny_local = None + self.x_lower_bound = None + self.x_upper_bound = None + self.y_lower_bound = None + self.y_upper_bound = None + self.cycleFreq = None + self.outFreq = None + self.regridOpt = None + self.timeInterpOpt = None + self.t2dDownscaleOpt = None + self.lapseGrid = None + self.rqiClimoGrid = None + self.swDowscaleOpt = None + self.precipDownscaleOpt = None + self.nwmPRISM_numGrid = None + self.nwmPRISM_denGrid = None + self.q2dDownscaleOpt = None + self.psfcDownscaleOpt = None + self.t2dBiasCorrectOpt = None + self.swBiasCorrectOpt = None + self.precipBiasCorrectOpt = None + self.q2dBiasCorrectOpt = None + self.windBiasCorrectOpt = None + self.psfcBiasCorrectOpt = None + self.lwBiasCorrectOpt = None + self.esmf_lats = None + self.esmf_lons = None + self.esmf_grid_in = None + self.regridComplete = False + self.regridObj = None + self.esmf_field_in = None + self.esmf_field_out = None # -------------------------------- # Only used for CFSv2 bias correction # as bias correction needs to take @@ -77,34 +79,35 @@ def __init__(self): self.coarse_input_forcings1 = None self.coarse_input_forcings2 = None # -------------------------------- - self.regridded_forcings1 = None - self.regridded_forcings2 = None - self.globalPcpRate1 = None - self.globalPcpRate2 = None - self.regridded_mask = None - self.final_forcings = None - self.ndv = None - self.file_in1 = None - self.file_in2 = None - self.fcst_hour1 = None - self.fcst_hour2 = None - self.fcst_date1 = None - self.fcst_date2 = None - self.height = None - self.netcdf_var_names = None - self.grib_mes_idx = None - self.input_map_output = None - self.grib_levels = None - self.grib_vars = None - self.tmpFile = None - self.tmpFileHeight = None - self.psfcTmp = None - self.t2dTmp = None - self.rstFlag = 0 - self.regridded_precip1 = None - self.regridded_precip2 = None - self.border = None - self.skip = False + self.skip = False + self.regridded_forcings1 = None + self.regridded_forcings2 = None + self.globalPcpRate1 = None + self.globalPcpRate2 = None + self.regridded_mask = None + self.final_forcings = None + self.ndv = None + self.file_in1 = None + self.file_in2 = None + self.fcst_hour1 = None + self.fcst_hour2 = None + self.fcst_date1 = None + self.fcst_date2 = None + self.height = None + self.netcdf_var_names = None + self.grib_mes_idx = None + self.input_map_output = None + self.grib_levels = None + self.grib_vars = None + self.tmpFile = None + self.tmpFileHeight = None + self.psfcTmp = None + self.t2dTmp = None + self.rstFlag = 0 + self.regridded_precip1 = None + self.regridded_precip2 = None + self.border = None + self.forcingInputModYaml = None def define_product(self): """ @@ -112,286 +115,45 @@ def define_product(self): forcing key value. :return: """ - GRIB1 = self.GRIB1 - GRIB2 = self.GRIB2 + GRIB1 = self.GRIB1 + GRIB2 = self.GRIB2 NETCDF = self.NETCDF - product_names = { - 1: "NLDAS2_GRIB1", - 2: "NARR_GRIB1", - 3: "GFS_Production_GRIB2", - 4: "NAM_Conus_Nest_GRIB2", - 5: "HRRR_Conus_GRIB2", - 6: "RAP_Conus_GRIB2", - 7: "CFSv2_6Hr_Global_GRIB2", - 8: "WRF_ARW_Hawaii_GRIB2", - 9: "GFS_Production_025d_GRIB2", - 10: "Custom_NetCDF_Hourly", - 11: "Custom_NetCDF_Hourly", - 12: "AORC", - 13: "NAM_Nest_3km_Hawaii", - 14: "NAM_Nest_3km_PuertoRico", - 15: "NAM_Nest_3km_Alaska", - 16: "NAM_Nest_3km_Hawaii_Radiation-Only", - 17: "NAM_Nest_3km_PuertoRico_Radiation-Only", - 18: "WRF_ARW_PuertoRico_GRIB2", - 19: "HRRR_Alaska_GRIB2", - 20: "Alaska_ExtAnA" - } - self.productName = product_names[self.keyValue] + yaml_stream = None + try: + yaml_stream = open(self.forcingInputModYaml) + config = yaml.safe_load(yaml_stream) + except yaml.YAMLError as yaml_exc: + err_handler.err_out_screen('Error parsing the configuration file: %s\n%s' % (self.forcingInputModYaml,yaml_exc)) + except IOError: + err_handler.err_out_screen('Unable to open the configuration file: %s' % self.forcingInputModYaml) + finally: + if yaml_stream: + yaml_stream.close() + try: + inputs = config[self.keyValue] + except KeyError: + err_handler.err_out_screen('Unable to locate Input map in configuration file.') + self.productName = inputs["product_name"] - ## DEFINED BY CONFIG - # product_types = { - # 1: GRIB1, - # 2: GRIB1, - # 3: GRIB2, - # 4: GRIB2, - # 5: GRIB2, - # 6: GRIB2, - # 7: GRIB2, - # 8: GRIB2, - # 9: GRIB2, - # 10: NETCDF, - # 11: NETCDF, - # 12: NETCDF, - # 13: GRIB2, - # 14: GRIB2, - # 15: GRIB2, - # 16: GRIB2, - # 17: GRIB2, - # 18: GRIB2, - # 19: GRIB2, - # 20: NETCDF - # } - # self.fileType = product_types[self.keyValue] - if self.fileType == 'GRIB1': - self.file_ext = '.grb' + if self.fileType == 'GRIB1': + self.file_ext = '.grb' elif self.fileType == 'GRIB2': - self.file_ext = '.grib2' + self.file_ext = '.grib2' elif self.fileType == 'NETCDF': - self.file_ext = '.nc' + self.file_ext = '.nc' - cycle_freq_minutes = { - 1: 60, - 2: 180, - 3: 360, - 4: 360, - 5: 60, - 6: 60, - 7: 360, - 8: 1440, - 9: 360, - 10: -9999, - 11: -9999, - 12: -9999, - 13: 360, - 14: 360, - 15: 360, - 16: 360, - 17: 360, - 18: 1440, - 19: 180, - 20: 60 - } - self.cycleFreq = cycle_freq_minutes[self.keyValue] + self.cycleFreq = inputs["cycle_freq_minutes"] - grib_vars_in = { - 1: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 2: None, - 3: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 4: None, - 5: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', - 'DLWRF', 'PRES'], - 6: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', - 'DLWRF', 'PRES'], - 7: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 8: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'], - 9: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 10: None, - 11: None, - 12: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', - 'DSWRF', 'DLWRF', 'PRES'], - 13: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 14: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 15: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 16: ['DSWRF', 'DLWRF'], - 17: ['DSWRF', 'DLWRF'], - 18: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'], - 19: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', - 'DLWRF', 'PRES'], - 20: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', - 'Q2D', 'PSFC', 'SWDOWN'] - } - self.grib_vars = grib_vars_in[self.keyValue] + self.grib_vars = inputs["grib_vars_in"] - grib_levels_in = { - 1: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 2: None, - 3: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 4: None, - 5: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 6: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 7: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 8: ['80 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface'], - 9: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 10: None, - 11: None, - 12: None, - 13: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 14: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 15: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 16: ['surface', 'surface'], - 17: ['surface', 'surface'], - 18: ['80 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface'], - 19: ['2 m above ground', '2 m above ground', - '10 m above ground', '10 m above ground', - 'surface', 'surface', 'surface', 'surface'], - 20: None - } - self.grib_levels = grib_levels_in[self.keyValue] + self.grib_levels = inputs["grib_levels_in"] - netcdf_variables = { - 1: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 2: None, - 3: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 4: None, - 5: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 6: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 7: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 8: ['TMP_80maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'PRES_surface'], - 9: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 10: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 11: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', - 'DLWRF', 'PRES'], - 12: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 13: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 14: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 15: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 16: ['DSWRF_surface', 'DLWRF_surface'], - 17: ['DSWRF_surface', 'DLWRF_surface'], - 18: ['TMP_80maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'PRES_surface'], - 19: ['TMP_2maboveground', 'SPFH_2maboveground', - 'UGRD_10maboveground', 'VGRD_10maboveground', - 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', - 'PRES_surface'], - 20: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', - 'Q2D', 'PSFC', 'SWDOWN'] - } - self.netcdf_var_names = netcdf_variables[self.keyValue] + self.netcdf_var_names = inputs["netcdf_variables"] - # arrays that store the message ids of required forcing variables for each forcing type - # TODO fill these arrays for forcing types other than GFS - grib_message_idx = { - 1: None, - 2: None, - 3: None, - 4: None, - 5: None, - 6: None, - 7: None, - 8: None, - 9: [33,34,39,40,43,88,91,6], - 10: None, - 11: None, - 12: None, - 13: None, - 14: None, - 15: None, - 16: None, - 17: None, - 18: None, - 19: None, - 20: None - } - self.grib_mes_idx = grib_message_idx[self.keyValue] + self.grib_mes_idx = inputs["grib_message_idx"] - input_map_to_outputs = { - 1: [4,5,0,1,3,7,2,6], - 2: None, - 3: [4,5,0,1,3,7,2,6], - 4: None, - 5: [4,5,0,1,3,7,2,6], - 6: [4,5,0,1,3,7,2,6], - 7: [4,5,0,1,3,7,2,6], - 8: [4,5,0,1,3,6], - 9: [4,5,0,1,3,7,2,6], - 10: [4,5,0,1,3,7,2,6], - 11: [4,5,0,1,3,7,2,6], - 12: [4,5,0,1,3,7,2,6], - 13: [4,5,0,1,3,7,2,6], - 14: [4,5,0,1,3,7,2,6], - 15: [4,5,0,1,3,7,2,6], - 16: [7,2], - 17: [7, 2], - 18: [4, 5, 0, 1, 3, 6], - 19: [4,5,0,1,3,7,2,6], - 20: [0,1,2,3,4,5,6,7] - } - self.input_map_output = input_map_to_outputs[self.keyValue] + self.input_map_output = inputs["input_map_to_outputs"] def calc_neighbor_files(self,ConfigOptions,dCurrent,MpiConfig): """ @@ -404,25 +166,27 @@ def calc_neighbor_files(self,ConfigOptions,dCurrent,MpiConfig): """ # First calculate the current input cycle date this # WRF-Hydro output timestep corresponds to. + ForcingEnum = ConfigOptions.ForcingEnum + find_neighbor_files = { - 1: time_handling.find_nldas_neighbors, - 3: time_handling.find_gfs_neighbors, - 5: time_handling.find_conus_hrrr_neighbors, - 6: time_handling.find_conus_rap_neighbors, - 7: time_handling.find_cfsv2_neighbors, - 8: time_handling.find_hourly_wrf_arw_neighbors, - 9: time_handling.find_gfs_neighbors, - 10: time_handling.find_custom_hourly_neighbors, - 11: time_handling.find_custom_hourly_neighbors, - 12: time_handling.find_aorc_neighbors, - 13: time_handling.find_nam_nest_neighbors, - 14: time_handling.find_nam_nest_neighbors, - 15: time_handling.find_nam_nest_neighbors, - 16: time_handling.find_nam_nest_neighbors, - 17: time_handling.find_nam_nest_neighbors, - 18: time_handling.find_hourly_wrf_arw_neighbors, - 19: time_handling.find_ak_hrrr_neighbors, - 20: time_handling.find_ak_ext_ana_neighbors, + str(ForcingEnum.NLDAS.name) : time_handling.find_nldas_neighbors, + str(ForcingEnum.GFS_GLOBAL.name) : time_handling.find_gfs_neighbors, + str(ForcingEnum.HRRR.name) : time_handling.find_conus_hrrr_neighbors, + str(ForcingEnum.RAP.name) : time_handling.find_conus_rap_neighbors, + str(ForcingEnum.CFS_V2.name) : time_handling.find_cfsv2_neighbors, + str(ForcingEnum.WRF_NEST_HI.name) : time_handling.find_hourly_wrf_arw_neighbors, + str(ForcingEnum.GFS_GLOBAL_25.name) : time_handling.find_gfs_neighbors, + str(ForcingEnum.CUSTOM_1.name) : time_handling.find_custom_hourly_neighbors, + str(ForcingEnum.CUSTOM_2.name) : time_handling.find_custom_hourly_neighbors, + str(ForcingEnum.CUSTOM_3.name) : time_handling.find_aorc_neighbors, + str(ForcingEnum.NAM_NEST_HI.name) : time_handling.find_nam_nest_neighbors, + str(ForcingEnum.NAM_NEST_PR.name) : time_handling.find_nam_nest_neighbors, + str(ForcingEnum.NAM_NEST_AK.name) : time_handling.find_nam_nest_neighbors, + str(ForcingEnum.NAM_NEST_HI_RAD.name): time_handling.find_nam_nest_neighbors, + str(ForcingEnum.NAM_NEST_PR_RAD.name): time_handling.find_nam_nest_neighbors, + str(ForcingEnum.WRF_ARW_PR.name) : time_handling.find_hourly_wrf_arw_neighbors, + str(ForcingEnum.HRRR_AK.name) : time_handling.find_ak_hrrr_neighbors, + str(ForcingEnum.HRRR_AK_EXT.name) : time_handling.find_ak_ext_ana_neighbors, } find_neighbor_files[self.keyValue](self, ConfigOptions, dCurrent,MpiConfig) @@ -439,25 +203,28 @@ def regrid_inputs(self, ConfigOptions, wrfHyroGeoMeta, MpiConfig): """ # Establish a mapping dictionary that will point the # code to the functions to that will regrid the data. + from core import regrid + ForcingEnum = ConfigOptions.ForcingEnum + regrid_inputs = { - 1: regrid.regrid_conus_rap, - 3: regrid.regrid_gfs, - 5: regrid.regrid_conus_hrrr, - 6: regrid.regrid_conus_rap, - 7: regrid.regrid_cfsv2, - 8: regrid.regrid_hourly_wrf_arw, - 9: regrid.regrid_gfs, - 10: regrid.regrid_custom_hourly_netcdf, - 11: regrid.regrid_custom_hourly_netcdf, - 12: regrid.regrid_custom_hourly_netcdf, - 13: regrid.regrid_nam_nest, - 14: regrid.regrid_nam_nest, - 15: regrid.regrid_nam_nest, - 16: regrid.regrid_nam_nest, - 17: regrid.regrid_nam_nest, - 18: regrid.regrid_hourly_wrf_arw, - 19: regrid.regrid_conus_hrrr, - 20: regrid.regrid_ak_ext_ana + str(ForcingEnum.NLDAS.name) : regrid.regrid_conus_rap, + str(ForcingEnum.GFS_GLOBAL.name) : regrid.regrid_gfs, + str(ForcingEnum.HRRR.name) : regrid.regrid_conus_hrrr, + str(ForcingEnum.RAP.name) : regrid.regrid_conus_rap, + str(ForcingEnum.CFS_V2.name) : regrid.regrid_cfsv2, + str(ForcingEnum.WRF_NEST_HI.name) : regrid.regrid_hourly_wrf_arw, + str(ForcingEnum.GFS_GLOBAL_25.name) : regrid.regrid_gfs, + str(ForcingEnum.CUSTOM_1.name) : regrid.regrid_custom_hourly_netcdf, + str(ForcingEnum.CUSTOM_2.name) : regrid.regrid_custom_hourly_netcdf, + str(ForcingEnum.CUSTOM_3.name) : regrid.regrid_custom_hourly_netcdf, + str(ForcingEnum.NAM_NEST_HI.name) : regrid.regrid_nam_nest, + str(ForcingEnum.NAM_NEST_PR.name) : regrid.regrid_nam_nest, + str(ForcingEnum.NAM_NEST_AK.name) : regrid.regrid_nam_nest, + str(ForcingEnum.NAM_NEST_HI_RAD.name): regrid.regrid_nam_nest, + str(ForcingEnum.NAM_NEST_PR_RAD.name): regrid.regrid_nam_nest, + str(ForcingEnum.WRF_ARW_PR.name) : regrid.regrid_hourly_wrf_arw, + str(ForcingEnum.HRRR_AK.name) : regrid.regrid_conus_hrrr, + str(ForcingEnum.HRRR_AK_EXT.name) : regrid.regrid_ak_ext_ana } regrid_inputs[self.keyValue](self,ConfigOptions,wrfHyroGeoMeta,MpiConfig) @@ -472,10 +239,11 @@ def temporal_interpolate_inputs(self,ConfigOptions,MpiConfig): :param MpiConfig: :return: """ + from core import timeInterpMod temporal_interpolate_inputs = { - 0: timeInterpMod.no_interpolation, - 1: timeInterpMod.nearest_neighbor, - 2: timeInterpMod.weighted_average + str(TemporalInterpEnum.NONE.name) : timeInterpMod.no_interpolation, + str(TemporalInterpEnum.NEAREST_NEIGHBOR.name) : timeInterpMod.nearest_neighbor, + str(TemporalInterpEnum.LINEAR_WEIGHT_AVG.name): timeInterpMod.weighted_average } temporal_interpolate_inputs[self.timeInterpOpt](self,ConfigOptions,MpiConfig) @@ -526,6 +294,7 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): InputDict[force_key].inDir = ConfigOptions.input_force_dirs[force_tmp] InputDict[force_key].paramDir = ConfigOptions.dScaleParamDirs[force_tmp] InputDict[force_key].fileType = ConfigOptions.input_force_types[force_tmp] + InputDict[force_key].forcingInputModYaml = ConfigOptions.forcingInputModYaml InputDict[force_key].define_product() InputDict[force_key].userFcstHorizon = ConfigOptions.fcst_input_horizons[force_tmp] InputDict[force_key].userCycleOffset = ConfigOptions.fcst_input_offsets[force_tmp] @@ -534,7 +303,7 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): # If we have specified specific humidity downscaling, establish arrays to hold # temporary temperature arrays that are un-downscaled. - if InputDict[force_key].q2dDownscaleOpt > 0: + if InputDict[force_key].q2dDownscaleOpt != DownScaleHumidEnum.NONE.name: InputDict[force_key].t2dTmp = np.empty([GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local], np.float32) @@ -546,7 +315,7 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): # of the local grid for this forcing, for a specific output timesetp. # This grid will be updated from one output timestep to another, and # also through downscaling and bias correction. - InputDict[force_key].final_forcings = np.empty([8,GeoMetaWrfHydro.ny_local, + InputDict[force_key].final_forcings = np.empty([len(ConfigOptions.OutputEnum),GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local], np.float64) InputDict[force_key].height = np.empty([GeoMetaWrfHydro.ny_local, @@ -555,7 +324,8 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): GeoMetaWrfHydro.nx_local],np.float32) # Obtain custom input cycle frequencies - if force_key == 10 or force_key == 11: + ForcingEnum = ConfigOptions.ForcingEnum + if force_key == ForcingEnum.CUSTOM_1.name or force_key == ForcingEnum.CUSTOM_2.name: InputDict[force_key].cycleFreq = ConfigOptions.customFcstFreq[custom_count] custom_count = custom_count + 1 diff --git a/core/forecastMod.py b/core/forecastMod.py index 59d3d3e..75cbe1b 100755 --- a/core/forecastMod.py +++ b/core/forecastMod.py @@ -123,7 +123,7 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo show_message = True for outStep in range(1, ConfigOptions.num_output_steps + 1): # Reset out final grids to missing values. - OutputObj.output_local[:, :, :] = -9999.0 + OutputObj.output_local[:, :, :] = ConfigOptions.globalNdv ConfigOptions.current_output_step = outStep OutputObj.outDate = ConfigOptions.current_fcst_cycle + datetime.timedelta( @@ -183,11 +183,9 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo # Regrid forcings. input_forcings.regrid_inputs(ConfigOptions, wrfHydroGeoMeta, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Run check on regridded fields for reasonable values that are not missing values. err_handler.check_forcing_bounds(ConfigOptions, input_forcings, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the # next set of forcings as the previous step just regridded the previous forcing. if input_forcings.rstFlag == 1: @@ -196,41 +194,36 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo # Set the forcings back to reflect we just regridded the previous set of inputs, not the next. input_forcings.regridded_forcings1[:, :, :] = \ input_forcings.regridded_forcings2[:, :, :] - # Re-calculate the neighbor files. input_forcings.calc_neighbor_files(ConfigOptions, OutputObj.outDate, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Regrid the forcings for the end of the window. input_forcings.regrid_inputs(ConfigOptions, wrfHydroGeoMeta, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) input_forcings.rstFlag = 0 - # Run temporal interpolation on the grids. input_forcings.temporal_interpolate_inputs(ConfigOptions, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Run bias correction. bias_correction.run_bias_correction(input_forcings, ConfigOptions, wrfHydroGeoMeta, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Run downscaling on grids for this output timestep. downscale.run_downscaling(input_forcings, ConfigOptions, wrfHydroGeoMeta, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Layer in forcings from this product. layeringMod.layer_final_forcings(OutputObj, input_forcings, ConfigOptions, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) ConfigOptions.currentForceNum = ConfigOptions.currentForceNum + 1 - if forceKey == 10: + if forceKey == "CUSTOM_1": ConfigOptions.currentCustomForceNum = ConfigOptions.currentCustomForceNum + 1 + + - else: # Process supplemental precipitation if we specified in the configuration file. if ConfigOptions.number_supp_pcp > 0: for suppPcpKey in ConfigOptions.supp_precip_forcings: diff --git a/core/geoMod.py b/core/geoMod.py index eda307c..722df13 100755 --- a/core/geoMod.py +++ b/core/geoMod.py @@ -1,6 +1,11 @@ import math -import ESMF +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + import numpy as np from netCDF4 import Dataset diff --git a/core/ioMod.py b/core/ioMod.py index 03bc447..05ef921 100755 --- a/core/ioMod.py +++ b/core/ioMod.py @@ -10,30 +10,30 @@ import os import shutil import subprocess - +import yaml +from strenum import StrEnum import numpy as np from netCDF4 import Dataset - from core import err_handler - +from core.enumConfig import RegriddingOptEnum class OutputObj: """ Abstract class to hold local "slabs" of final output grids. """ - def __init__(self,GeoMetaWrfHydro): + def __init__(self,GeoMetaWrfHydro,ConfigOptions): self.output_local = None self.outPath = None self.outDate = None - self.out_ndv = -9999 + self.out_ndv = -999999 # Create local "slabs" to hold final output grids. These # will be collected during the output routine below. - self.output_local = np.empty([9, GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local]) + self.output_local = np.empty([len(ConfigOptions.OutputEnum), GeoMetaWrfHydro.ny_local, GeoMetaWrfHydro.nx_local]) #self.output_local[:,:,:] = self.out_ndv - def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): + def output_final_ldasin(self, ConfigOptions, geoMetaWrfHydro, MpiConfig): """ Output routine to produce final LDASIN files for the WRF-Hydro modeling system. This function is assuming all regridding, @@ -50,19 +50,20 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): :param MpiConfig: :return: """ - output_variable_attribute_dict = { - 'U2D': [0,'m s-1','x_wind','10-m U-component of wind','time: point',0.001,0.0,3], - 'V2D': [1,'m s-1','y_wind','10-m V-component of wind','time: point',0.001,0.0,3], - 'LWDOWN': [2,'W m-2','surface_downward_longwave_flux', - 'Surface downward long-wave radiation flux','time: point',0.001,0.0,3], - 'RAINRATE': [3,'mm s^-1','precipitation_flux','Surface Precipitation Rate','time: mean',1.0,0.0,0], - 'T2D': [4,'K','air_temperature','2-m Air Temperature','time: point',0.01,100.0,2], - 'Q2D': [5,'kg kg-1','surface_specific_humidity','2-m Specific Humidity','time: point',0.000001,0.0,6], - 'PSFC': [6,'Pa','air_pressure','Surface Pressure','time: point',0.1,0.0,1], - 'SWDOWN': [7,'W m-2','surface_downward_shortwave_flux', - 'Surface downward short-wave radiation flux','time: point',0.001,0.0,3] - } + yaml_stream = None + try: + yaml_stream = open(ConfigOptions.outputVarAttrYaml) + config = yaml.safe_load(yaml_stream) + except yaml.YAMLError as yaml_exc: + err_handler.err_out_screen('Error parsing the configuration file: %s\n%s' % (ConfigOptions.outputVarAttrYaml,yaml_exc)) + except IOError: + err_handler.err_out_screen('Unable to open the configuration file: %s' % ConfigOptions.outputVarAttrYaml) + finally: + if yaml_stream: + yaml_stream.close() + output_variable_attribute_dict = config + if ConfigOptions.include_lqfraq: output_variable_attribute_dict['LQFRAQ'] = [8, '%', 'liquid_water_fraction', 'Fraction of precipitation that is liquid vs. frozen', @@ -70,11 +71,11 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): # Compose the ESMF remapped string attribute based on the regridding option chosen by the user. # We will default to the regridding method chosen for the first input forcing selected. - if ConfigOptions.regrid_opt[0] == 1: + if ConfigOptions.regrid_opt[0] == str(RegriddingOptEnum.ESMF_BILINEAR.name): regrid_att = "remapped via ESMF regrid_with_weights: Bilinear" - elif ConfigOptions.regrid_opt[0] == 2: + elif ConfigOptions.regrid_opt[0] == str(RegriddingOptEnum.ESMF_NEAREST_NEIGHBOR.name): regrid_att = "remapped via ESMF regrid_with_weights: Nearest Neighbor" - elif ConfigOptions.regrid_opt[0] == 3: + elif ConfigOptions.regrid_opt[0] == str(RegriddingOptEnum.ESMF_CONSERVATIVE_BILINEAR.name): regrid_att = "remapped via ESMF regrid_with_weights: Conservative Bilinear" # Ensure all processors are synced up before outputting. @@ -125,10 +126,11 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): err_handler.log_critical(ConfigOptions, MpiConfig) break try: - model_init_time = ConfigOptions.current_fcst_cycle - if ConfigOptions.ana_flag == 1: - model_init_time -= datetime.timedelta(seconds=ConfigOptions.output_freq * 60) - idOut.model_initialization_time = model_init_time.strftime("%Y-%m-%d_%H:%M:00") + if ConfigOptions.ana_flag: + model_init = ConfigOptions.b_date_proc - datetime.timedelta(minutes=ConfigOptions.output_freq) + else: + model_init = ConfigOptions.current_fcst_cycle + idOut.model_initialization_time = model_init.strftime("%Y-%m-%d_%H:%M:00") except: ConfigOptions.errMsg = "Unable to set the model_initialization_time global " \ "attribute in: " + self.outPath @@ -161,12 +163,18 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): break try: - idOut.model_total_valid_times = float(ConfigOptions.actual_output_steps) + idOut.model_total_valid_times = np.int32(ConfigOptions.actual_output_steps) except: ConfigOptions.errMsg = "Unable to create total_valid_times global attribute in: " + self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) break + if ConfigOptions.spatial_meta is not None: + # apply spatial_global_atts to output globals attrs + for k, v in geoMetaWrfHydro.spatial_global_atts.items(): + if k not in ("Source_Software", "history", "processing_notes", "version"): # don't add these + idOut.setncattr(k, v) + # Create variables. try: idOut.createVariable('time','i4',('time')) @@ -312,13 +320,13 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): zlib = True complevel = 2 least_significant_digit = None if varTmp == 'RAINRATE' else \ - output_variable_attribute_dict[varTmp][7] # use all digits in RAINRATE + output_variable_attribute_dict[varTmp]['least_significant_digit'] # use all digits in RAINRATE else: zlib = False complevel = 0 least_significant_digit = None - if ConfigOptions.useFloats or varTmp == 'RAINRATE': # RAINRATE always a float + if ConfigOptions.useFloats == 'FLOAT' or varTmp == 'RAINRATE': # RAINRATE always a float fill_value = ConfigOptions.globalNdv dtype = 'f4' else: @@ -338,7 +346,7 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): err_handler.log_critical(ConfigOptions, MpiConfig) break try: - idOut.variables[varTmp].cell_methods = output_variable_attribute_dict[varTmp][4] + idOut.variables[varTmp].cell_methods = output_variable_attribute_dict[varTmp]['cell_methods'] except: ConfigOptions.errMsg = "Unable to create cell_methods attribute for: " + varTmp + \ " in: " + self.outPath @@ -378,38 +386,38 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): break try: - idOut.variables[varTmp].units = output_variable_attribute_dict[varTmp][1] + idOut.variables[varTmp].units = output_variable_attribute_dict[varTmp]['units'] except: ConfigOptions.errMsg = "Unable to create units attribute for: " + varTmp + " in: " + \ self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) break try: - idOut.variables[varTmp].standard_name = output_variable_attribute_dict[varTmp][2] + idOut.variables[varTmp].standard_name = output_variable_attribute_dict[varTmp]['standard_name'] except: ConfigOptions.errMsg = "Unable to create standard_name attribute for: " + varTmp + \ " in: " + self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) break try: - idOut.variables[varTmp].long_name = output_variable_attribute_dict[varTmp][3] + idOut.variables[varTmp].long_name = output_variable_attribute_dict[varTmp]['long_name'] except: ConfigOptions.errMsg = "Unable to create long_name attribute for: " + varTmp + \ " in: " + self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) break # If we are using scale_factor / add_offset, create here. - if not ConfigOptions.useFloats: + if not ConfigOptions.useFloats == 'FLOAT': if varTmp != 'RAINRATE': try: - idOut.variables[varTmp].scale_factor = output_variable_attribute_dict[varTmp][5] + idOut.variables[varTmp].scale_factor = output_variable_attribute_dict[varTmp]['scale_factor'] except (ValueError, IOError): ConfigOptions.errMsg = "Unable to create scale_factor attribute for: " + varTmp + \ " in: " + self.outPath err_handler.log_critical(ConfigOptions, MpiConfig) break try: - idOut.variables[varTmp].add_offset = output_variable_attribute_dict[varTmp][6] + idOut.variables[varTmp].add_offset = output_variable_attribute_dict[varTmp]['add_offset'] except (ValueError, IOError): ConfigOptions.errMsg = "Unable to create add_offset attribute for: " + varTmp + \ " in: " + self.outPath @@ -434,9 +442,8 @@ def output_final_ldasin(self,ConfigOptions,geoMetaWrfHydro,MpiConfig): # final = MpiConfig.comm.gather(self.output_local[output_variable_attribute_dict[varTmp][0],:,:],root=0) # Use gatherv to merge the data slabs - dataOutTmp = MpiConfig.merge_slabs_gatherv(self.output_local[output_variable_attribute_dict[varTmp][0],:,:], ConfigOptions) - except Exception as e: - print(e) + dataOutTmp = MpiConfig.merge_slabs_gatherv(self.output_local[output_variable_attribute_dict[varTmp]['ind'],:,:], ConfigOptions) + except OSError: ConfigOptions.errMsg = "Unable to gather final grids for: " + varTmp err_handler.log_critical(ConfigOptions, MpiConfig) continue @@ -508,7 +515,6 @@ def open_grib2(GribFileIn,NetCdfFileOut,Wgrib2Cmd,ConfigOptions,MpiConfig, "Multi-sensor estimated precipitation accumulation 1-hour:mm\n" ) os.environ['GRIB2TABLE'] = g2path - exitcode = subprocess.call(Wgrib2Cmd, shell=True) #print("exitcode: " + str(exitcode)) @@ -529,6 +535,11 @@ def open_grib2(GribFileIn,NetCdfFileOut,Wgrib2Cmd,ConfigOptions,MpiConfig, err = None exitcode = None + # remove temporary grib2.tbl file + g2path = os.path.join(ConfigOptions.scratch_dir, "grib2.tbl") + if os.path.isfile(g2path): + os.remove(g2path) + # Ensure file exists. if not os.path.isfile(NetCdfFileOut): ConfigOptions.errMsg = "Expected NetCDF file: " + NetCdfFileOut + \ @@ -562,7 +573,6 @@ def open_grib2(GribFileIn,NetCdfFileOut,Wgrib2Cmd,ConfigOptions,MpiConfig, err_handler.log_warning(ConfigOptions, MpiConfig) # idTmp = None pass - if idTmp is not None and inputVar is not None: # Loop through all the expected variables. if inputVar not in idTmp.variables.keys(): @@ -573,7 +583,6 @@ def open_grib2(GribFileIn,NetCdfFileOut,Wgrib2Cmd,ConfigOptions,MpiConfig, pass else: idTmp = None - # Ensure all processors are synced up before outputting. # MpiConfig.comm.barrier() ## THIS HAPPENS IN check_program_status diff --git a/core/layeringMod.py b/core/layeringMod.py index fa90c2e..ca9787a 100755 --- a/core/layeringMod.py +++ b/core/layeringMod.py @@ -19,18 +19,13 @@ def layer_final_forcings(OutputObj,input_forcings,ConfigOptions,MpiConfig): :param MpiConfig: :return: """ - # Loop through the 8 forcing products to layer in: - # 0.) U-Wind (m/s) - # 1.) V-Wind (m/s) - # 2.) Surface incoming longwave radiation flux (W/m^2) - # 3.) Precipitation rate (mm/s) - # 4.) 2-meter temperature (K) - # 5.) 2-meter specific humidity (kg/kg) - # 6.) Surface pressure (Pa) - # 7.) Surface incoming shortwave radiation flux (W/m^2) + OutputEnum = ConfigOptions.OutputEnum + # Loop through the forcing products to layer in - for force_idx in range(0,8): - if force_idx in input_forcings.input_map_output: + for force_id in OutputEnum: #len of OutputEnum class + force_name = force_id.name + if force_name in input_forcings.input_map_output: + force_idx = input_forcings.input_map_output.index(force_name) outLayerCurrent = OutputObj.output_local[force_idx,:,:] layerIn = input_forcings.final_forcings[force_idx,:,:] indSet = np.where(layerIn != ConfigOptions.globalNdv) @@ -53,9 +48,17 @@ def layer_supplemental_forcing(OutputObj, supplemental_precip, ConfigOptions, Mp :param MpiConfig: :return: """ + + OutputEnum = ConfigOptions.OutputEnum indSet = np.where(supplemental_precip.final_supp_precip != ConfigOptions.globalNdv) layerIn = supplemental_precip.final_supp_precip - layerOut = OutputObj.output_local[supplemental_precip.output_var_idx, :, :] + for ind, xvar in enumerate(OutputEnum): + if xvar.name == supplemental_precip.output_var_idx: + outId = ind + break + + + layerOut = OutputObj.output_local[outId, :, :] #TODO: review test layering for ExtAnA calculation to replace FE QPE with MPE RAINRATE #If this isn't sufficient, replace QPE with MPE here: @@ -70,5 +73,5 @@ def layer_supplemental_forcing(OutputObj, supplemental_precip, ConfigOptions, Mp layerOut = layerOut # TODO: test that even does anything...?s - OutputObj.output_local[supplemental_precip.output_var_idx, :, :] = layerOut + OutputObj.output_local[outId, :, :] = layerOut diff --git a/core/parallel.py b/core/parallel.py index 910b578..f93ef8d 100755 --- a/core/parallel.py +++ b/core/parallel.py @@ -140,7 +140,7 @@ def scatter_array_scatterv_no_cache(self,geoMeta,src_array,ConfigOptions): data_type_flag = 1 if src_array.dtype == np.float64: data_type_flag = 2 - if src_array.dtype == np.bool: + if src_array.dtype == bool: data_type_flag = 3 # Broadcast the data_type_flag to other processors @@ -206,7 +206,7 @@ def scatter_array_scatterv_no_cache(self,geoMeta,src_array,ConfigOptions): recvbuf=np.empty([counts[self.rank]],np.float32) elif data_type_flag == 3: data_type = MPI.BOOL - recvbuf = np.empty([counts[self.rank]], np.bool) + recvbuf = np.empty([counts[self.rank]], bool) else: data_type = MPI.DOUBLE recvbuf = np.empty([counts[self.rank]], np.float64) diff --git a/core/regrid.py b/core/regrid.py index c6d7ae8..8cb1342 100755 --- a/core/regrid.py +++ b/core/regrid.py @@ -2,17 +2,21 @@ """ Regridding module file for regridding input forcing files. """ +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF import os import sys import traceback import time - -import ESMF import numpy as np - +from strenum import StrEnum from core import err_handler from core import ioMod from core import timeInterpMod +#from core.forcingInputMod import OutputEnum NETCDF = "NETCDF" GRIB2 = "GRIB2" @@ -83,7 +87,7 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co except: config_options.errMsg = f"Unable to open input NetCDF file: {input_forcings.file_in}" err_handler.log_critical(config_options, mpi_config) - + ds.set_auto_scale(True) ds.set_auto_mask(False) @@ -122,11 +126,11 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) # Create out regridded numpy arrays to hold the regridded data. - input_forcings.regridded_forcings1 = np.empty([8, wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], + input_forcings.regridded_forcings1 = np.empty([len(config_options.OutputEnum), wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], np.float32) - input_forcings.regridded_forcings2 = np.empty([8, wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], + input_forcings.regridded_forcings2 = np.empty([len(config_options.OutputEnum), wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], np.float32) - + for force_count, nc_var in enumerate(input_forcings.netcdf_var_names): var_tmp = None if mpi_config.rank == 0: @@ -139,13 +143,18 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = f"Unable to extract: {nc_var} from: {input_forcings.file_in2} ({str(err)})" err_handler.log_critical(config_options, mpi_config) - + err_handler.check_program_status(config_options, mpi_config) var_sub_tmp = mpi_config.scatter_array(input_forcings, var_tmp, config_options) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break + try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = var_sub_tmp + input_forcings.regridded_forcings2[outId, :, :] = var_sub_tmp except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to extract ExtAnA forcing data from the AK AnA field: " + str(err) err_handler.log_critical(config_options, mpi_config) @@ -153,9 +162,9 @@ def regrid_ak_ext_ana(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] - + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] + if mpi_config.rank == 0: ds.close() @@ -190,7 +199,7 @@ def _regrid_ak_ext_ana_pcp_stage4(supplemental_precip, config_options, wrf_hydro lat_var = "latitude" lon_var = "longitude" - + if supplemental_precip.fileType != NETCDF: # This file shouldn't exist.... but if it does (previously failed # execution of the program), remove it..... @@ -276,7 +285,7 @@ def _regrid_ak_ext_ana_pcp_stage4(supplemental_precip, config_options, wrf_hydro # Convert the 6-hourly precipitation total to a rate of mm/s try: ind_valid = np.where(supplemental_precip.regridded_precip2 != config_options.globalNdv) - supplemental_precip.regridded_precip2[ind_valid] = supplemental_precip.regridded_precip2[ind_valid] / 21600.0 + supplemental_precip.regridded_precip2[ind_valid] = supplemental_precip.regridded_precip2[ind_valid] / 3600.0 del ind_valid except (ValueError, ArithmeticError, AttributeError, KeyError) as npe: config_options.errMsg = "Unable to run NDV search on STAGE IV supplemental precipitation: " + str(npe) @@ -518,6 +527,10 @@ def regrid_conus_hrrr(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break # Set any pixel cells outside the input domain to the global missing value. try: input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ @@ -528,7 +541,7 @@ def regrid_conus_hrrr(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to extract regridded HRRR forcing data from the ESMF field: " + str(err) @@ -538,8 +551,8 @@ def regrid_conus_hrrr(input_forcings, config_options, wrf_hydro_geo_meta, mpi_co # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] # mpi_config.comm.barrier() # Close the temporary NetCDF file and remove it. @@ -747,6 +760,10 @@ def regrid_conus_rap(input_forcings, config_options, wrf_hydro_geo_meta, mpi_con err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break # Set any pixel cells outside the input domain to the global missing value. try: input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ @@ -758,7 +775,7 @@ def regrid_conus_rap(input_forcings, config_options, wrf_hydro_geo_meta, mpi_con err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to place RAP ESMF data into local array: " + str(err) @@ -768,15 +785,15 @@ def regrid_conus_rap(input_forcings, config_options, wrf_hydro_geo_meta, mpi_con # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # Close the temporary NetCDF file and remove it. @@ -975,6 +992,10 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) var_sub_tmp = mpi_config.scatter_array(input_forcings, var_tmp, config_options) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break # Assign local CFSv2 data to the input forcing object.. IF..... we are running the # bias correction. These grids are interpolated in a separate routine, AFTER bias # correction has taken place. @@ -984,7 +1005,7 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) # np.any(input_forcings.coarse_input_forcings2) and \ # ConfigOptions.current_output_step == 1: # We need to create NumPy arrays to hold the CFSv2 global data. - input_forcings.coarse_input_forcings1 = np.empty([8, var_sub_tmp.shape[0], var_sub_tmp.shape[1]], + input_forcings.coarse_input_forcings1 = np.empty([len(config_options.OutputEnum), var_sub_tmp.shape[0], var_sub_tmp.shape[1]], np.float64) if input_forcings.coarse_input_forcings2 is None: # and config_options.current_output_step == 1: @@ -992,11 +1013,11 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) # np.any(input_forcings.coarse_input_forcings2) and \ # ConfigOptions.current_output_step == 1: # We need to create NumPy arrays to hold the CFSv2 global data. - input_forcings.coarse_input_forcings2 = np.empty([8, var_sub_tmp.shape[0], var_sub_tmp.shape[1]], + input_forcings.coarse_input_forcings2 = np.empty([len(config_options.OutputEnum), var_sub_tmp.shape[0], var_sub_tmp.shape[1]], np.float64) try: - input_forcings.coarse_input_forcings2[input_forcings.input_map_output[force_count], :, :] = var_sub_tmp + input_forcings.coarse_input_forcings2[outId, :, :] = var_sub_tmp except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to place local CFSv2 input variable: " + \ input_forcings.netcdf_var_names[force_count] + \ @@ -1005,13 +1026,17 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) # print("DEBUG: ", input_forcings.coarse_input_forcings2, input_forcings.input_map_output, force_count) if config_options.current_output_step == 1: - input_forcings.coarse_input_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.coarse_input_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.coarse_input_forcings1[outId, :, :] = \ + input_forcings.coarse_input_forcings2[outId, :, :] else: input_forcings.coarse_input_forcings2 = None input_forcings.coarse_input_forcings1 = None err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break # Only regrid the current files if we did not specify the NLDAS2 NWM bias correction, which needs to take place # first before any regridding can take place. That takes place in the bias-correction routine. if not config_options.runCfsNldasBiasCorrect: @@ -1042,7 +1067,7 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to extract ESMF field data for CFSv2: " + str(err) @@ -1052,14 +1077,14 @@ def regrid_cfsv2(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config) # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) else: # Set regridded arrays to dummy values as they are regridded later in the bias correction routine. - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings1[outId, :, :] = \ config_options.globalNdv - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ config_options.globalNdv # Close the temporary NetCDF file and remove it. @@ -1233,8 +1258,13 @@ def regrid_custom_hourly_netcdf(input_forcings, config_options, wrf_hydro_geo_me err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break + try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to place local ESMF regridded data into local array: " + str(err) @@ -1244,8 +1274,8 @@ def regrid_custom_hourly_netcdf(input_forcings, config_options, wrf_hydro_geo_me # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # Close the NetCDF file @@ -1289,7 +1319,8 @@ def regrid_gfs(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): # check / set previous file to see if we're going to reuse reuse_prev_file = (input_forcings.file_in2 == regrid_gfs.last_file) regrid_gfs.last_file = input_forcings.file_in2 - + print(input_forcings.file_in2) + print(regrid_gfs.last_file) # This file may exist. If it does, and we don't need it again, remove it..... if not reuse_prev_file and mpi_config.rank == 0: if os.path.isfile(input_forcings.tmpFile): @@ -1497,6 +1528,10 @@ def regrid_gfs(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break # Set any pixel cells outside the input domain to the global missing value. try: input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ @@ -1508,7 +1543,7 @@ def regrid_gfs(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to extract GFS ESMF field data to local array: " + str(err) @@ -1518,8 +1553,8 @@ def regrid_gfs(input_forcings, config_options, wrf_hydro_geo_meta, mpi_config): # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # Close the temporary NetCDF file and remove it. @@ -1726,6 +1761,11 @@ def regrid_nam_nest(input_forcings, config_options, wrf_hydro_geo_meta, mpi_conf err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break + # Set any pixel cells outside the input domain to the global missing value. try: input_forcings.esmf_field_out.data[np.where(input_forcings.regridded_mask == 0)] = \ @@ -1736,7 +1776,7 @@ def regrid_nam_nest(input_forcings, config_options, wrf_hydro_geo_meta, mpi_conf err_handler.check_program_status(config_options, mpi_config) try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to place local ESMF regridded data into local array: " + str(err) @@ -1746,8 +1786,8 @@ def regrid_nam_nest(input_forcings, config_options, wrf_hydro_geo_meta, mpi_conf # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # Close the temporary NetCDF file and remove it. @@ -1992,15 +2032,15 @@ def regrid_mrms_hourly(supplemental_precip, config_options, wrf_hydro_geo_meta, config_options.statusMsg = "MRMS Will not be filtered using RQI values." err_handler.log_msg(config_options, mpi_config) - elif supplemental_precip.rqiMethod == 2: + elif supplemental_precip.rqiMethod == "NWM": # Read in the RQI field from monthly climatological files. ioMod.read_rqi_monthly_climo(config_options, mpi_config, supplemental_precip, wrf_hydro_geo_meta) - elif supplemental_precip.rqiMethod == 1: + elif supplemental_precip.rqiMethod == "MRMS": # We are using the MRMS RQI field in realtime supplemental_precip.regridded_rqi2[:, :] = supplemental_precip.esmf_field_out.data err_handler.check_program_status(config_options, mpi_config) - if supplemental_precip.rqiMethod == 1: + if supplemental_precip.rqiMethod == "MRMS": # Close the temporary NetCDF file and remove it. if mpi_config.rank == 0: try: @@ -2044,9 +2084,13 @@ def regrid_mrms_hourly(supplemental_precip, config_options, wrf_hydro_geo_meta, # Set any pixel cells outside the input domain to the global missing value, and set negative precip values to 0 try: + if len(np.argwhere(supplemental_precip.esmf_field_out.data < 0)) > 0: + supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = config_options.globalNdv + # config_options.statusMsg = "WARNING: Found negative precipitation values in MRMS data, setting to missing_value" + # err_handler.log_warning(config_options, mpi_config) + supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.regridded_mask == 0)] = \ config_options.globalNdv - supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.esmf_field_out.data < 0)] = 0 except (ValueError, ArithmeticError) as npe: config_options.errMsg = "Unable to run mask search on MRMS supplemental precip: " + str(npe) @@ -2317,8 +2361,13 @@ def regrid_hourly_wrf_arw(input_forcings, config_options, wrf_hydro_geo_meta, mp err_handler.log_critical(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + for ind, xvar in enumerate(config_options.OutputEnum): + if xvar.name == input_forcings.input_map_output[force_count]: + outId = ind + break + try: - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] = \ input_forcings.esmf_field_out.data except (ValueError, KeyError, AttributeError) as err: config_options.errMsg = "Unable to place local ESMF regridded data into local array: " + str(err) @@ -2328,8 +2377,8 @@ def regrid_hourly_wrf_arw(input_forcings, config_options, wrf_hydro_geo_meta, mp # If we are on the first timestep, set the previous regridded field to be # the latest as there are no states for time 0. if config_options.current_output_step == 1: - input_forcings.regridded_forcings1[input_forcings.input_map_output[force_count], :, :] = \ - input_forcings.regridded_forcings2[input_forcings.input_map_output[force_count], :, :] + input_forcings.regridded_forcings1[outId, :, :] = \ + input_forcings.regridded_forcings2[outId, :, :] err_handler.check_program_status(config_options, mpi_config) # Close the temporary NetCDF file and remove it. @@ -2594,7 +2643,7 @@ def regrid_sbcv2_liquid_water_fraction(supplemental_forcings, config_options, wr err_handler.check_program_status(config_options, mpi_config) -def regrid_hourly_nbm_apcp(supplemental_precip, config_options, wrf_hydro_geo_meta, mpi_config): +def regrid_hourly_nbm(forcings_or_precip, config_options, wrf_hydro_geo_meta, mpi_config): """ Function for handling regridding hourly forecasted NBM precipitation. :param supplemental_precip: @@ -2606,31 +2655,32 @@ def regrid_hourly_nbm_apcp(supplemental_precip, config_options, wrf_hydro_geo_me # Do we want to use NBM data at this timestep? If not, log and continue if not config_options.use_data_at_current_time: if mpi_config.rank == 0: - config_options.statusMsg = "Exceeded max hours for NBM precipitation, will not use NBM in final layering." + config_options.statusMsg = "Exceeded max hours for NBM data, will not use NBM in final layering." err_handler.log_msg(config_options, mpi_config) return - + # If the expected file is missing, this means we are allowing missing files, simply # exit out of this routine as the regridded fields have already been set to NDV. - if not os.path.exists(supplemental_precip.file_in1): + if not os.path.exists(forcings_or_precip.file_in1): return # Check to see if the regrid complete flag for this # output time step is true. This entails the necessary # inputs have already been regridded and we can move on. - if supplemental_precip.regridComplete: + if forcings_or_precip.regridComplete: return nbm_tmp_nc = config_options.scratch_dir + "/NBM_PCP_TMP-{}.nc".format(mkfilename()) - if os.path.isfile(nbm_tmp_nc): - config_options.statusMsg = "Found old temporary file: " + \ - nbm_tmp_nc + " - Removing....." - err_handler.log_warning(config_options, mpi_config) - try: - os.remove(nbm_tmp_nc) - except OSError: - config_options.errMsg = "Unable to remove file: " + nbm_tmp_nc - err_handler.log_critical(config_options, mpi_config) + + if mpi_config.rank == 0: + if os.path.isfile(nbm_tmp_nc): + config_options.statusMsg = "Found old temporary file: " + nbm_tmp_nc + " - Removing....." + err_handler.log_warning(config_options, mpi_config) + try: + os.remove(nbm_tmp_nc) + except OSError: + config_options.errMsg = "Unable to remove file: " + nbm_tmp_nc + err_handler.log_critical(config_options, mpi_config) # Perform a GRIB dump to NetCDF for the precip data. fieldnbm_match1 = "\":APCP:\"" @@ -2640,91 +2690,213 @@ def regrid_hourly_nbm_apcp(supplemental_precip, config_options, wrf_hydro_geo_me + " -match " + fieldnbm_match2 \ + " -not " + fieldnbm_notmatch1 \ + " -netcdf " + nbm_tmp_nc - id_tmp = ioMod.open_grib2(supplemental_precip.file_in1, nbm_tmp_nc, cmd1, config_options, - mpi_config, supplemental_precip.netcdf_var_names[0]) + + + id_tmp = ioMod.open_grib2(supplemental_precip.file_in1, nbm_tmp_nc, cmd1, config_options, mpi_config, supplemental_precip.netcdf_var_names[0]) err_handler.check_program_status(config_options, mpi_config) - # Check to see if we need to calculate regridding weights. - calc_regrid_flag = check_supp_pcp_regrid_status(id_tmp, supplemental_precip, config_options, - wrf_hydro_geo_meta, mpi_config) + if forcings_or_precip.grib_vars is not None: + fields = [] + for force_count, grib_var in enumerate(forcings_or_precip.grib_vars): + if mpi_config.rank == 0: + config_options.statusMsg = "Converting NBM Variable: " + grib_var + err_handler.log_msg(config_options, mpi_config) + time_str = "{}-{} hour acc fcst".format(forcings_or_precip.fcst_hour1, forcings_or_precip.fcst_hour2) \ + if grib_var == 'APCP' else str(forcings_or_precip.fcst_hour2) + " hour fcst" + fields.append(':' + grib_var + ':' + + forcings_or_precip.grib_levels[force_count] + ':' + + time_str + ":") + # fields.append(":(HGT):(surface):") + # Create a temporary NetCDF file from the GRIB2 file. + cmd = '$WGRIB2 -match "(' + '|'.join(fields) + ')" -not "prob" -not "ens" ' + \ + forcings_or_precip.file_in1 + " -netcdf " + nbm_tmp_nc + else: + # Perform a GRIB dump to NetCDF for the precip data. + fieldnbm_match1 = "\":APCP:\"" + fieldnbm_match2 = "\"" + str(forcings_or_precip.fcst_hour1) + "-" + str(forcings_or_precip.fcst_hour2) + "\"" + fieldnbm_notmatch1 = "\"prob\"" # We don't want the probabilistic QPF layers + cmd = "$WGRIB2 " + forcings_or_precip.file_in1 + " -match " + fieldnbm_match1 \ + + " -match " + fieldnbm_match2 \ + + " -not " + fieldnbm_notmatch1 \ + + " -netcdf " + nbm_tmp_nc + + id_tmp = ioMod.open_grib2(forcings_or_precip.file_in1, nbm_tmp_nc, cmd, config_options, + mpi_config, forcings_or_precip.netcdf_var_names[0]) + err_handler.check_program_status(config_options, mpi_config) - if calc_regrid_flag: + for force_count, nc_var in enumerate(forcings_or_precip.netcdf_var_names): if mpi_config.rank == 0: - config_options.statusMsg = "Calculating NBM regridding weights." + config_options.statusMsg = "Processing NBM Variable: " + nc_var err_handler.log_msg(config_options, mpi_config) - calculate_supp_pcp_weights(supplemental_precip, id_tmp, supplemental_precip.file_in1, config_options, mpi_config) + + # Check to see if we need to calculate regridding weights. + is_supp = forcings_or_precip.grib_vars is None + if is_supp: + tag = "supplemental precip" + calc_regrid_flag = check_supp_pcp_regrid_status(id_tmp, forcings_or_precip, config_options, + wrf_hydro_geo_meta, mpi_config) + else: + tag = "input" + calc_regrid_flag = check_regrid_status(id_tmp, force_count, forcings_or_precip, + config_options, wrf_hydro_geo_meta, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + if calc_regrid_flag: + if is_supp: + if mpi_config.rank == 0: + config_options.statusMsg = f"Calculating NBM {tag} regridding weights." + err_handler.log_msg(config_options, mpi_config) + calculate_supp_pcp_weights(forcings_or_precip, id_tmp, forcings_or_precip.file_in1, config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + else: + if mpi_config.rank == 0: + config_options.statusMsg = f"Calculating NBM {tag} regridding weights." + err_handler.log_msg(config_options, mpi_config) + calculate_weights(id_tmp, force_count, forcings_or_precip, config_options, mpi_config, fill=True) + err_handler.check_program_status(config_options, mpi_config) + + # Regrid the height variable. + if config_options.grid_meta is None: + config_options.errMsg = "No NBM height file supplied, downscaling will not be available" + err_handler.log_warning(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + else: + if not os.path.exists(config_options.grid_meta): + config_options.errMsg = "NBM height file \"{config_options.grid_meta}\" does not exist" + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + terrain_tmp = os.path.join(config_options.scratch_dir, 'nbm_terrain_temp.nc') + cmd = f"$WGRIB2 {config_options.grid_meta} -netcdf {terrain_tmp}" + hgt_tmp = ioMod.open_grib2(config_options.grid_meta, terrain_tmp, cmd, config_options, + mpi_config, 'DIST_surface') + if mpi_config.rank == 0: + var_tmp = hgt_tmp.variables['DIST_surface'][0, :, :] + else: + var_tmp = None + err_handler.check_program_status(config_options, mpi_config) + + var_sub_tmp = mpi_config.scatter_array(forcings_or_precip, var_tmp, config_options) + err_handler.check_program_status(config_options, mpi_config) + + try: + forcings_or_precip.esmf_field_in.data[:, :] = var_sub_tmp + except (ValueError, KeyError, AttributeError) as err: + config_options.errMsg = "Unable to place NBM elevation data into the ESMF field object: " \ + + str(err) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + if mpi_config.rank == 0: + config_options.statusMsg = "Regridding NBM elevation data to the WRF-Hydro domain." + err_handler.log_msg(config_options, mpi_config) + try: + forcings_or_precip.esmf_field_out = forcings_or_precip.regridObj(forcings_or_precip.esmf_field_in, + forcings_or_precip.esmf_field_out) + except ValueError as ve: + config_options.errMsg = "Unable to regrid NBM elevation data to the WRF-Hydro domain " \ + "using ESMF: " + str(ve) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + # Set any pixel cells outside the input domain to the global missing value. + try: + forcings_or_precip.esmf_field_out.data[np.where(forcings_or_precip.regridded_mask == 0)] = \ + config_options.globalNdv + except (ValueError, ArithmeticError) as npe: + config_options.errMsg = "Unable to compute mask on NBM elevation data: " + str(npe) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + try: + forcings_or_precip.height[:, :] = forcings_or_precip.esmf_field_out.data + except (ValueError, KeyError, AttributeError) as err: + config_options.errMsg = "Unable to extract ESMF regridded NBM elevation data to a local " \ + "array: " + str(err) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + if mpi_config.rank == 0: + hgt_tmp.close() + + + # Regrid the input variables. + var_tmp = None + if mpi_config.rank == 0: + config_options.statusMsg = f"Regridding NBM {nc_var}" + err_handler.log_msg(config_options, mpi_config) + try: + var_tmp = id_tmp.variables[nc_var][0, :, :] + except (ValueError, KeyError, AttributeError) as err: + config_options.errMsg = "Unable to extract data from NBM file: " + \ + forcings_or_precip.file_in1 + " (" + str(err) + ")" + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) + + var_sub_tmp = mpi_config.scatter_array(forcings_or_precip, var_tmp, config_options) err_handler.check_program_status(config_options, mpi_config) - # Regrid the input variables. - var_tmp = None - if mpi_config.rank == 0: - config_options.statusMsg = "Regridding NBM APCP Precipitation." - err_handler.log_msg(config_options, mpi_config) try: - var_tmp = id_tmp.variables['APCP_surface'][0, :, :] + forcings_or_precip.esmf_field_in.data[:, :] = var_sub_tmp except (ValueError, KeyError, AttributeError) as err: - config_options.errMsg = "Unable to extract precipitation from NBM file: " + \ - supplemental_precip.file_in1 + " (" + str(err) + ")" + config_options.errMsg = f"Unable to place NBM {tag} into local ESMF field: " + str(err) err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) - var_sub_tmp = mpi_config.scatter_array(supplemental_precip, var_tmp, config_options) - err_handler.check_program_status(config_options, mpi_config) + try: + forcings_or_precip.esmf_field_out = forcings_or_precip.regridObj(forcings_or_precip.esmf_field_in, + forcings_or_precip.esmf_field_out) + except ValueError as ve: + config_options.errMsg = f"Unable to regrid NBM {tag}: " + str(ve) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) - try: - supplemental_precip.esmf_field_in.data[:, :] = var_sub_tmp - except (ValueError, KeyError, AttributeError) as err: - config_options.errMsg = "Unable to place NBM precipitation into local ESMF field: " + str(err) - err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + # Set any pixel cells outside the input domain to the global missing value. + try: + forcings_or_precip.esmf_field_out.data[np.where(forcings_or_precip.regridded_mask == 0)] = \ + config_options.globalNdv + except (ValueError, ArithmeticError) as npe: + config_options.errMsg = "Unable to run mask search on NBM supplemental precipitation: " + str(npe) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) - try: - supplemental_precip.esmf_field_out = supplemental_precip.regridObj(supplemental_precip.esmf_field_in, - supplemental_precip.esmf_field_out) - except ValueError as ve: - config_options.errMsg = "Unable to regrid NBM supplemental precipitation: " + str(ve) - err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + destination1 = forcings_or_precip.regridded_precip1 if is_supp else \ + forcings_or_precip.regridded_forcings1[forcings_or_precip.input_map_output[force_count]] - # Set any pixel cells outside the input domain to the global missing value. - try: - supplemental_precip.esmf_field_out.data[np.where(supplemental_precip.regridded_mask == 0)] = \ - config_options.globalNdv - except (ValueError, ArithmeticError) as npe: - config_options.errMsg = "Unable to run mask search on NBM supplemental precipitation: " + str(npe) - err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + destination2 = forcings_or_precip.regridded_precip2 if is_supp else \ + forcings_or_precip.regridded_forcings2[forcings_or_precip.input_map_output[force_count]] - supplemental_precip.regridded_precip2[:, :] = supplemental_precip.esmf_field_out.data - err_handler.check_program_status(config_options, mpi_config) + destination2[:, :] = forcings_or_precip.esmf_field_out.data + err_handler.check_program_status(config_options, mpi_config) - # Convert the hourly precipitation total from kg.m-2.hour-1 to a rate of mm.s-1 - try: - ind_valid = np.where(supplemental_precip.regridded_precip2 != config_options.globalNdv) - if supplemental_precip.input_frequency == 60.0: - supplemental_precip.regridded_precip2[ind_valid] = supplemental_precip.regridded_precip2[ind_valid] / 3600.0 - elif supplemental_precip.input_frequency == 360.0: - supplemental_precip.regridded_precip2[ind_valid] = supplemental_precip.regridded_precip2[ind_valid] / 21600.0 #uniform disaggregation for 6-hourly nbm data - del ind_valid - except (ValueError, ArithmeticError, AttributeError, KeyError) as npe: - config_options.errMsg = "Unable to run NDV search on NBM supplemental precipitation: " + str(npe) - err_handler.log_critical(config_options, mpi_config) - err_handler.check_program_status(config_options, mpi_config) + # Convert the hourly precipitation total from kg.m-2.hour-1 to a rate of mm.s-1 + if 'APCP' in nc_var: + try: + ind_valid = np.where(destination2 != config_options.globalNdv) + if forcings_or_precip.input_frequency == 60.0: + destination2[ind_valid] = destination2[ind_valid] / 3600.0 + elif forcings_or_precip.input_frequency == 360.0: + destination2[ind_valid] = destination2[ind_valid] / 21600.0 # uniform disaggregation for 6-hourly nbm data + del ind_valid + except (ValueError, ArithmeticError, AttributeError, KeyError) as npe: + config_options.errMsg = "Unable to run NDV search on NBM precipitation: " + str(npe) + err_handler.log_critical(config_options, mpi_config) + err_handler.check_program_status(config_options, mpi_config) - # If we are on the first timestep, set the previous regridded field to be - # the latest as there are no states for time 0. - if config_options.current_output_step == 1: - supplemental_precip.regridded_precip1[:, :] = \ - supplemental_precip.regridded_precip2[:, :] - err_handler.check_program_status(config_options, mpi_config) + # If we are on the first timestep, set the previous regridded field to be + # the latest as there are no states for time 0. + if config_options.current_output_step == 1: + destination1[:, :] = \ + destination2[:, :] + err_handler.check_program_status(config_options, mpi_config) # Close the temporary NetCDF file and remove it. if mpi_config.rank == 0: try: id_tmp.close() except OSError: - config_options.errMsg = "Unable to close NetCDF file: " + supplemental_precip.file_in1 + config_options.errMsg = "Unable to close NetCDF file: " + forcings_or_precip.file_in1 err_handler.log_critical(config_options, mpi_config) try: os.remove(nbm_tmp_nc) @@ -2767,9 +2939,9 @@ def check_regrid_status(id_tmp, force_count, input_forcings, config_options, wrf if input_forcings.nx_global is None or input_forcings.ny_global is None: # This is the first timestep. # Create out regridded numpy arrays to hold the regridded data. - input_forcings.regridded_forcings1 = np.empty([8, wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], + input_forcings.regridded_forcings1 = np.empty([len(config_options.OutputEnum), wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], np.float32) - input_forcings.regridded_forcings2 = np.empty([8, wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], + input_forcings.regridded_forcings2 = np.empty([len(config_options.OutputEnum), wrf_hydro_geo_meta.ny_local, wrf_hydro_geo_meta.nx_local], np.float32) if mpi_config.rank == 0: @@ -2872,7 +3044,7 @@ def check_supp_pcp_regrid_status(id_tmp, supplemental_precip, config_options, wr def calculate_weights(id_tmp, force_count, input_forcings, config_options, mpi_config, - lat_var="latitude", lon_var="longitude"): + lat_var="latitude", lon_var="longitude", fill=False): """ Function to calculate ESMF weights based on the output ESMF field previously calculated, along with input lat/lon grids, @@ -3083,12 +3255,14 @@ def calculate_weights(id_tmp, force_count, input_forcings, config_options, mpi_c err_handler.log_msg(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) try: + extrap_method = ESMF.ExtrapMethod.CREEP_FILL if fill else ESMF.ExtrapMethod.NONE begin = time.monotonic() input_forcings.regridObj = ESMF.Regrid(input_forcings.esmf_field_in, input_forcings.esmf_field_out, src_mask_values=np.array([0, config_options.globalNdv]), regrid_method=ESMF.RegridMethod.BILINEAR, unmapped_action=ESMF.UnmappedAction.IGNORE, + extrap_method=extrap_method, filename=weight_file) end = time.monotonic() diff --git a/core/suppPrecipMod.py b/core/suppPrecipMod.py old mode 100755 new mode 100644 index 35d7d04..c977523 --- a/core/suppPrecipMod.py +++ b/core/suppPrecipMod.py @@ -7,7 +7,9 @@ from core import time_handling from core import regrid from core import timeInterpMod - +from strenum import StrEnum +from core.enumConfig import TemporalInterpEnum +import yaml class supplemental_precip: """ @@ -71,6 +73,7 @@ def __init__(self): self.global_x_upper = None self.global_y_upper = None self.has_cache = False + self.suppPrecipModYaml = None def define_product(self): """ @@ -78,30 +81,26 @@ def define_product(self): forcing key value. :return: """ - product_names = { - 1: "MRMS_1HR_Radar_Only", - 2: "MRMS_1HR_Gage_Corrected", - 3: "WRF_ARW_Hawaii_2p5km_PCP", - 4: "WRF_ARW_PuertoRico_2p5km_PCP", - 5: "CONUS_MRMS_1HR_MultiSensor", - 6: "Hawaii_MRMS_1HR_MultiSensor", - 7: "MRMS_LiquidWaterFraction", - 8: "NBM_CORE_CONUS_APCP", - 9: "NBM_CORE_ALASKA_APCP", - 10: "AK_MRMS", - 11: "AK_Stage_IV_Precip-MRMS" - } - self.productName = product_names[self.keyValue] - ## DEFINED IN CONFIG - # product_types = { - # 1: "GRIB2", - # 2: "GRIB2", - # 3: "GRIB2", - # 4: "GRIB2", - # 5: "GRIB2" - # } - # self.fileType = product_types[self.keyValue] + yaml_stream = None + try: + yaml_stream = open(self.suppPrecipModYaml) + config = yaml.safe_load(yaml_stream) + except yaml.YAMLError as yaml_exc: + err_handler.err_out_screen('Error parsing the configuration file: %s\n%s' % (self.suppPrecipModYaml,yaml_exc)) + except IOError: + err_handler.err_out_screen('Unable to open the configuration file: %s' % self.suppPrecipModYaml) + finally: + if yaml_stream: + yaml_stream.close() + + try: + inputs = config[self.keyValue] + except KeyError: + err_handler.err_out_screen('Unable to locate Input map in configuration file.') + + self.productName = inputs["product_name"] + if self.fileType == 'GRIB1': self.file_ext = '.grb' elif self.fileType == 'GRIB2': @@ -109,80 +108,15 @@ def define_product(self): elif self.fileType == 'NETCDF': self.file_ext = '.nc' - grib_vars_in = { - 1: None, - 2: None, - 3: None, - 4: None, - 5: None, - 6: None, - 7: None, - 8: None, - 9: None, - 10: None, - 11: None - } - self.grib_vars = grib_vars_in[self.keyValue] + self.grib_vars = inputs["grib_vars_in"] - grib_levels_in = { - 1: ['BLAH'], - 2: ['BLAH'], - 3: ['BLAH'], - 4: ['BLAH'], - 5: ['BLAH'], - 6: ['BLAH'], - 7: ['BLAH'], - 8: ['BLAH'], - 9: ['BLAH'], - 10: ['BLAH'], - 11: ['BLAH'] - } - self.grib_levels = grib_levels_in[self.keyValue] + self.grib_levels = inputs["grib_levels_in"] - netcdf_variables = { - 1: ['RadarOnlyQPE01H_0mabovemeansealevel'], - 2: ['GaugeCorrQPE01H_0mabovemeansealevel'], - 3: ['APCP_surface'], - 4: ['APCP_surface'], - 5: ['MultiSensorQPE01H_0mabovemeansealevel'], - 6: ['MultiSensorQPE01H_0mabovemeansealevel'], - 7: ['sbcv2_lwf'], - 8: ['APCP_surface'], - 9: ['APCP_surface'], - 10: ['MultiSensorQPE01H_0mabovemeansealevel'], - 11: [] #Set dynamically since we have have Stage IV and MRMS - } - self.netcdf_var_names = netcdf_variables[self.keyValue] + self.netcdf_var_names = inputs["netcdf_variables"] - netcdf_rqi_variables = { - 1: ['RadarQualityIndex_0mabovemeansealevel'], - 2: ['RadarQualityIndex_0mabovemeansealevel'], - 3: None, - 4: None, - 5: None, - 6: None, - 7: None, - 8: None, - 9: None, - 10: None, - 11: None - } - self.rqi_netcdf_var_names = netcdf_rqi_variables[self.keyValue] + self.rqi_netcdf_var_names = inputs["netcdf_rqi_variables"] - output_variables = { - 1: 3, # RAINRATE - 2: 3, - 3: 3, - 4: 3, - 5: 3, - 6: 3, - 7: 8, # LQFRAC - 8: 3, - 9: 3, - 10: 3, - 11: 3 - } - self.output_var_idx = output_variables[self.keyValue] + self.output_var_idx = inputs["output_variables"] def calc_neighbor_files(self,ConfigOptions,dCurrent,MpiConfig): """ @@ -195,29 +129,22 @@ def calc_neighbor_files(self,ConfigOptions,dCurrent,MpiConfig): """ # First calculate the current input cycle date this # WRF-Hydro output timestep corresponds to. + SuppForcingPcpEnum = ConfigOptions.SuppForcingPcpEnum find_neighbor_files = { - 1: time_handling.find_hourly_mrms_radar_neighbors, - 2: time_handling.find_hourly_mrms_radar_neighbors, - 3: time_handling.find_hourly_wrf_arw_neighbors, - 4: time_handling.find_hourly_wrf_arw_neighbors, - 5: time_handling.find_hourly_mrms_radar_neighbors, - 6: time_handling.find_hourly_mrms_radar_neighbors, - 7: time_handling.find_sbcv2_lwf_neighbors, - 8: time_handling.find_hourly_nbm_apcp_neighbors, - 9: time_handling.find_hourly_nbm_apcp_neighbors, - 10: time_handling.find_hourly_mrms_radar_neighbors, - 11: time_handling.find_ak_ext_ana_precip_neighbors + str(SuppForcingPcpEnum.MRMS.name) : time_handling.find_hourly_mrms_radar_neighbors, + str(SuppForcingPcpEnum.MRMS_GAGE.name) : time_handling.find_hourly_mrms_radar_neighbors, + str(SuppForcingPcpEnum.WRF_ARW_HI.name) : time_handling.find_hourly_wrf_arw_neighbors, + str(SuppForcingPcpEnum.WRF_ARW_PR.name) : time_handling.find_hourly_wrf_arw_neighbors, + str(SuppForcingPcpEnum.MRMS_CONUS_MS.name): time_handling.find_hourly_mrms_radar_neighbors, + str(SuppForcingPcpEnum.MRMS_HI_MS.name) : time_handling.find_hourly_mrms_radar_neighbors, + str(SuppForcingPcpEnum.MRMS_SBCV2.name) : time_handling.find_sbcv2_lwf_neighbors, + str(SuppForcingPcpEnum.AK_OPT1.name) : time_handling.find_hourly_nbm_neighbors, + str(SuppForcingPcpEnum.AK_OPT2.name) : time_handling.find_hourly_nbm_neighbors, + str(SuppForcingPcpEnum.AK_MRMS.name) : time_handling.find_hourly_mrms_radar_neighbors, + str(SuppForcingPcpEnum.AK_NWS_IV.name) : time_handling.find_ak_ext_ana_precip_neighbors } - find_neighbor_files[self.keyValue](self, ConfigOptions, dCurrent, MpiConfig) - # try: - # find_neighbor_files[self.keyValue](self,ConfigOptions,dCurrent,MpiConfig) - # except TypeError: - # ConfigOptions.errMsg = "Unable to execute find_neighbor_files for " \ - # "supplemental precipitation: " + self.productName - # raise - # except: - # raise + find_neighbor_files[SuppForcingPcpEnum(self.keyValue).name](self, ConfigOptions, dCurrent, MpiConfig) def regrid_inputs(self,ConfigOptions,wrfHyroGeoMeta,MpiConfig): """ @@ -231,26 +158,21 @@ def regrid_inputs(self,ConfigOptions,wrfHyroGeoMeta,MpiConfig): """ # Establish a mapping dictionary that will point the # code to the functions to that will regrid the data. + SuppForcingPcpEnum = ConfigOptions.SuppForcingPcpEnum regrid_inputs = { - 1: regrid.regrid_mrms_hourly, - 2: regrid.regrid_mrms_hourly, - 3: regrid.regrid_hourly_wrf_arw_hi_res_pcp, - 4: regrid.regrid_hourly_wrf_arw_hi_res_pcp, - 5: regrid.regrid_mrms_hourly, - 6: regrid.regrid_mrms_hourly, - 7: regrid.regrid_sbcv2_liquid_water_fraction, - 8: regrid.regrid_hourly_nbm_apcp, - 9: regrid.regrid_hourly_nbm_apcp, - 10: regrid.regrid_mrms_hourly, - 11: regrid.regrid_ak_ext_ana_pcp + str(SuppForcingPcpEnum.MRMS.name) : regrid.regrid_mrms_hourly, + str(SuppForcingPcpEnum.MRMS_GAGE.name) : regrid.regrid_mrms_hourly, + str(SuppForcingPcpEnum.WRF_ARW_HI.name) : regrid.regrid_hourly_wrf_arw_hi_res_pcp, + str(SuppForcingPcpEnum.WRF_ARW_PR.name) : regrid.regrid_hourly_wrf_arw_hi_res_pcp, + str(SuppForcingPcpEnum.MRMS_CONUS_MS.name): regrid.regrid_mrms_hourly, + str(SuppForcingPcpEnum.MRMS_HI_MS.name) : regrid.regrid_mrms_hourly, + str(SuppForcingPcpEnum.MRMS_SBCV2.name) : regrid.regrid_sbcv2_liquid_water_fraction, + str(SuppForcingPcpEnum.AK_OPT1.name) : regrid.regrid_hourly_nbm, + str(SuppForcingPcpEnum.AK_OPT2.name) : regrid.regrid_hourly_nbm, + str(SuppForcingPcpEnum.AK_MRMS.name) : regrid.regrid_mrms_hourly, + str(SuppForcingPcpEnum.AK_NWS_IV.name) : regrid.regrid_ak_ext_ana_pcp } regrid_inputs[self.keyValue](self,ConfigOptions,wrfHyroGeoMeta,MpiConfig) - #try: - # regrid_inputs[self.keyValue](self,ConfigOptions,MpiConfig) - #except: - # ConfigOptions.errMsg = "Unable to execute regrid_inputs for " + \ - # "input forcing: " + self.productName - # raise def temporal_interpolate_inputs(self,ConfigOptions,MpiConfig): """ @@ -264,19 +186,12 @@ def temporal_interpolate_inputs(self,ConfigOptions,MpiConfig): :return: """ temporal_interpolate_inputs = { - 0: timeInterpMod.no_interpolation_supp_pcp, - 1: timeInterpMod.nearest_neighbor_supp_pcp, - 2: timeInterpMod.weighted_average_supp_pcp + str(TemporalInterpEnum.NONE.name) : timeInterpMod.no_interpolation_supp_pcp, + str(TemporalInterpEnum.NEAREST_NEIGHBOR.name) : timeInterpMod.nearest_neighbor_supp_pcp, + str(TemporalInterpEnum.LINEAR_WEIGHT_AVG.name): timeInterpMod.weighted_average_supp_pcp } - temporal_interpolate_inputs[self.timeInterpOpt](self,ConfigOptions,MpiConfig) - #temporal_interpolate_inputs[self.keyValue](self,ConfigOptions,MpiConfig) - #try: - # temporal_interpolate_inputs[self.timeInterpOpt](self,ConfigOptions,MpiConfig) - #except: - # ConfigOptions.errMsg = "Unable to execute temporal_interpolate_inputs " + \ - # " for input forcing: " + self.productName - # raise - + temporal_interpolate_inputs[TemporalInterpEnum(self.timeInterpOpt).name](self,ConfigOptions,MpiConfig) + def initDict(ConfigOptions,GeoMetaWrfHydro): """ Initial function to create an supplemental dictionary, which @@ -298,6 +213,7 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): InputDict[supp_pcp_key].inDir = ConfigOptions.supp_precip_dirs[supp_pcp_tmp] InputDict[supp_pcp_key].fileType = ConfigOptions.supp_precip_file_types[supp_pcp_tmp] + InputDict[supp_pcp_key].suppPrecipModYaml = ConfigOptions.suppPrecipModYaml InputDict[supp_pcp_key].define_product() # Initialize the local final grid of values @@ -317,4 +233,3 @@ def initDict(ConfigOptions,GeoMetaWrfHydro): InputDict[supp_pcp_key].rqiThresh = 1.0 return InputDict - diff --git a/core/tests/assets/style.css b/core/tests/assets/style.css new file mode 100644 index 0000000..e80e4e8 --- /dev/null +++ b/core/tests/assets/style.css @@ -0,0 +1,177 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; +} + +h1 { + font-size: 24px; + color: black; +} + +h2 { + font-size: 16px; + color: black; +} + +p { + color: black; +} + +a { + color: #999; +} + +table { + border-collapse: collapse; +} + +/****************************** + * SUMMARY INFORMATION + ******************************/ + +#environment td { + padding: 5px; + border: 1px solid #E6E6E6; +} + +#environment tr:nth-child(odd) { + background-color: #f6f6f6; +} + +/****************************** + * TEST RESULT COLORS + ******************************/ +span.passed, .passed .col-result { + color: green; +} +span.skipped, span.xfailed, span.rerun, .skipped .col-result, .xfailed .col-result, .rerun .col-result { + color: orange; +} +span.error, span.failed, span.xpassed, .error .col-result, .failed .col-result, .xpassed .col-result { + color: red; +} + + +/****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ + +/*------------------ + * 1. Table Layout + *------------------*/ + +#results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100% +} + +#results-table th, #results-table td { + padding: 5px; + border: 1px solid #E6E6E6; + text-align: left +} +#results-table th { + font-weight: bold +} + +/*------------------ + * 2. Extra + *------------------*/ + +.log:only-child { + height: inherit +} +.log { + background-color: #e6e6e6; + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + height: 230px; + overflow-y: scroll; + padding: 5px; + white-space: pre-wrap +} +div.image { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin-left: 5px; + overflow: hidden; + width: 320px +} +div.image img { + width: 320px +} +div.video { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin-left: 5px; + overflow: hidden; + width: 320px +} +div.video video { + overflow: hidden; + width: 320px; + height: 240px; +} +.collapsed { + display: none; +} +.expander::after { + content: " (show details)"; + color: #BBB; + font-style: italic; + cursor: pointer; +} +.collapser::after { + content: " (hide details)"; + color: #BBB; + font-style: italic; + cursor: pointer; +} + +/*------------------ + * 3. Sorting items + *------------------*/ +.sortable { + cursor: pointer; +} + +.sort-icon { + font-size: 0px; + float: left; + margin-right: 5px; + margin-top: 5px; + /*triangle*/ + width: 0; + height: 0; + border-left: 8px solid transparent; + border-right: 8px solid transparent; +} + +.inactive .sort-icon { + /*finish triangle*/ + border-top: 8px solid #E6E6E6; +} + +.asc.active .sort-icon { + /*finish triangle*/ + border-bottom: 8px solid #999; +} + +.desc.active .sort-icon { + /*finish triangle*/ + border-top: 8px solid #999; +} diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py new file mode 100644 index 0000000..f87e21a --- /dev/null +++ b/core/tests/fixtures.py @@ -0,0 +1,43 @@ +import pytest + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro + +import numpy as np + + +@pytest.fixture +def mpi_meta(config_options): + config_options.read_config() + + mpi_meta = MpiConfig() + mpi_meta.initialize_comm(config_options) + + return mpi_meta + +@pytest.fixture +def geo_meta(config_options, mpi_meta): + config_options.read_config() + + WrfHydroGeoMeta = GeoMetaWrfHydro() + WrfHydroGeoMeta.initialize_destination_geo(config_options, mpi_meta) + WrfHydroGeoMeta.initialize_geospatial_metadata(config_options, mpi_meta) + + return WrfHydroGeoMeta + +@pytest.fixture +def config_options(): + config_path = './yaml/configOptions_valid.yaml' + config_options = ConfigOptions(config_path) + config_options.read_config() + yield config_options + +@pytest.fixture +def config_options_cfs(): + config_path = './yaml/configOptions_cfs.yaml' + config_options = ConfigOptions(config_path) + config_options.read_config() + yield config_options + + diff --git a/core/tests/gdrive_download.py b/core/tests/gdrive_download.py new file mode 100644 index 0000000..5a4646a --- /dev/null +++ b/core/tests/gdrive_download.py @@ -0,0 +1,72 @@ +from argparse import ArgumentParser + +import requests +import os + + +def maybe_download_testdata(): + # See if we've downloaded the test data from + # google drive yet, and download it if not + if os.path.isdir("config_data"): + return + + id = "1qZbSpgUbi6Is5VlJzsvZ3R5w1WtA1uS3" + outfile = "config_data.tar.gz" + + download_file_from_google_drive(id, outfile) + os.system("tar xzf %s" % outfile) + os.remove(outfile) + + +def download_file_from_google_drive(id, destination): + print('downloading google drive file id ' + id + ' to ' + destination) + URL = "https://docs.google.com/uc?export=download" + + session = requests.Session() + + response = session.get(URL, params={'id': id, 'alt': 'media', 'confirm':'t'} + , stream=True) + token = get_confirm_token(response) + + if token: + params = {'id': id, 'confirm': token} + response = session.get(URL, params=params, stream=True) + + save_response_content(response, destination) + + +def get_confirm_token(response): + for key, value in response.cookies.items(): + if key.startswith('download_warning'): + return value + + return None + + +def save_response_content(response, destination): + CHUNK_SIZE = 32768 + + with open(destination, "wb") as f: + for chunk in response.iter_content(CHUNK_SIZE): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + +def main(): + + parser = ArgumentParser() + parser.add_argument("--file_id", + dest="file_id", + help="Google drive file ID. Get from shareable link") + parser.add_argument("--dest_file", + dest="dest_file", + help="Full path including filename for downloaded file.") + + args = parser.parse_args() + file_id = args.file_id + dest_file = args.dest_file + + download_file_from_google_drive(file_id, dest_file) + +if __name__ == "__main__": + main() diff --git a/core/tests/run_tests.py b/core/tests/run_tests.py new file mode 100644 index 0000000..8be98ed --- /dev/null +++ b/core/tests/run_tests.py @@ -0,0 +1,65 @@ +from gdrive_download import maybe_download_testdata +import subprocess + +COLOR_GREEN = "\033[92m" +COLOR_NORM = "\033[0m" +COLOR_RED = "\033[91m" + + +def run(): + maybe_download_testdata() + + failed = 0 + + print ("Testing without mpirun...") + + cmd = "python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s----Tests not using mpirun failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- Tests not using mpirun passed%s" % (COLOR_GREEN, COLOR_NORM)) + + print ("Testing with 1 MPI process...") + + cmd = "mpirun -n 1 python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 1 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 1 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + print ("Testing with 4 MPI processes...") + + cmd = "mpirun -n 4 python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 4 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 4 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + # print ("Testing with 8 MPI processes...") + + cmd = "mpirun -n 8 python -m pytest --with-mpi" + test_proc = subprocess.run(cmd, shell=True) + if test_proc.returncode != 0: + print("%s---- MPI nprocs = 8 tests failed!!!%s" % (COLOR_RED, COLOR_NORM)) + failed += 1 + else: + print("%s---- MPI nprocs = 8 tests passed%s" % (COLOR_GREEN, COLOR_NORM)) + + if failed > 0: + print("%s---- Total %s MPI configurations failed!!!%s" % (failed, COLOR_RED, COLOR_NORM)) + exit(-1) + + print("%s******** All tests passed ***********%s" % (COLOR_GREEN, COLOR_NORM)) + exit(0) + + + + +if __name__ == "__main__": + run() diff --git a/core/tests/test_bias_correction.py b/core/tests/test_bias_correction.py new file mode 100644 index 0000000..232adba --- /dev/null +++ b/core/tests/test_bias_correction.py @@ -0,0 +1,685 @@ +import pytest +import numpy as np +from datetime import datetime + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro +from core.forcingInputMod import * +from core.enumConfig import * + +from fixtures import * +import core.bias_correction as BC + + +def date(datestr): + return datetime.strptime(datestr, '%Y-%m-%d %H:%M:%S') + +var_list = { + 'T2D': 300, + 'Q2D': 50.0, + 'U2D': 2.5, + 'V2D': 1.5, + 'RAINRATE': 2.5, + 'SWDOWN': 400, + 'LWDOWN': 400, + 'PSFC': 90000 +} + +expected_values = { + 'no_bias_correct': [ + { 'index': (0,0), 'value': 1.5, 'tolerance': None} + ], + 'ncar_tbl_correction': [ + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 1.85, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 1.85, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 0}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 1.6, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 1.6, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 6}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 1.52, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 1.52, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 12}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'T2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 2.5, 'tolerance': None, 'var': 'Q2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 1.45, 'tolerance': None, 'var': 'U2D', 'fcst_hour': 18}, + { 'index': (0,0), 'value': 1.45, 'tolerance': None, 'var': 'V2D', 'fcst_hour': 18} + ], + 'ncar_blanket_adjustment_lw': [ + { 'index': (0,0), 'value': 9.82194214668789, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 7.763851170382089, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 6.8580578533121095, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 8.91614882961791, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 7.479010060854788, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 4.6895996292499, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 6.160989939145213, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 8.950400370750101, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_sw_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.5826259963214397, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4999999948202003, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4999999948202003, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.540613193064928, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.4394609276205301, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.500000003795177, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.500000003795177, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.4702432015910745, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_temp_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.462288117950048, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.49941014460614, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.5757118820499518, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.53858985539386, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 0.9518785484037021, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.8684798631054194, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.884121451596298, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 0.9675201368945807, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_temp_gfs_bias_correct': [ + { 'index': (0,0), 'value': 2.6484931133084233, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 2.1467845464398003, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 0.23150688669157682, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 0.7332154535601993, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'ncar_lwdown_gfs_bias_correct': [ + { 'index': (0,0), 'value': 10.897517774766143, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 12.813333511002988, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 11.902482225233857, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 9.986666488997013, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'ncar_wspd_hrrr_bias_correct': [ + { 'index': (0,0), 'value': 1.7324061943600948, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.6027854243986395, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.592862924985717, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.7224836949471725, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 1}, + { 'index': (0,0), 'value': 1.2131465997822362, 'tolerance': None, 'date': date('2023-01-01 00:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.080473211999704, 'tolerance': None, 'date': date('2023-04-01 06:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.1504572971498712, 'tolerance': None, 'date': date('2023-07-01 12:00:00'), 'ana': 0}, + { 'index': (0,0), 'value': 1.2831306849324031, 'tolerance': None, 'date': date('2023-10-01 18:00:00'), 'ana': 0} + ], + 'ncar_wspd_gfs_bias_correct': [ + { 'index': (0,0), 'value': 1.560235849440387, 'tolerance': None, 'date': date('2023-01-01 00:00:00')}, + { 'index': (0,0), 'value': 1.2559415578811088, 'tolerance': None, 'date': date('2023-04-01 06:00:00')}, + { 'index': (0,0), 'value': 1.1569214380849937, 'tolerance': None, 'date': date('2023-07-01 12:00:00')}, + { 'index': (0,0), 'value': 1.4612157296442718, 'tolerance': None, 'date': date('2023-10-01 18:00:00')} + ], + 'cfsv2_nldas_nwm_bias_correct': [ + { 'index': (0,0), 'value': 285, 'tolerance': None, 'var': 'T2D' }, + { 'index': (0,0), 'value': 0.009524456432886543, 'tolerance': None, 'var': 'Q2D' }, + { 'index': (0,0), 'value': 3.562956632266148, 'tolerance': None, 'var': 'U2D' }, + { 'index': (0,0), 'value': 0.11510974533140078, 'tolerance': None, 'var': 'V2D' }, + { 'index': (0,0), 'value': 0.01622283237712793, 'tolerance': None, 'var': 'RAINRATE' }, + { 'index': (0,0), 'value': 0, 'tolerance': None, 'var': 'SWDOWN' }, + { 'index': (0,0), 'value': 272, 'tolerance': None, 'var': 'LWDOWN' }, + { 'index': (0,0), 'value': 49999 , 'tolerance': None, 'var': 'PSFC' } + ] +} + +####################################################### +# +# This test suite creates a mock 'forcing grid' filled +# with a known value, calls each bias_correction function, +# and samples a gridpoint to compare against the expected +# bias-corrected value for that function. +# +# The functions currently expect an exact (full precision) +# match, but a tolerance can be set for each test or +# globally in the run_and_compare() function +# +####################################################### + +@pytest.fixture +def unbiased_input_forcings(config_options,geo_meta): + """ + Prepare the mock input forcing grid + """ + config_options.read_config() + + inputForcingMod = initDict(config_options,geo_meta) + input_forcings = inputForcingMod[config_options.input_forcings[0]] + + # put a value of 1.5 at every gridpoint, for 10 mock variables + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 + + return input_forcings + +@pytest.fixture +def unbiased_input_forcings_cfs(config_options_cfs,geo_meta,mpi_meta): + """ + Prepare the mock input forcing grid for CFS + """ + # Don't run for mpi processes > 0, since the + # CFS and NLDAS parameters are spatially dependent and we're not sure + # how we are splitting up the grid. + if mpi_meta.size > 1: + return + config_options_cfs.read_config() + + inputForcingMod = initDict(config_options_cfs,geo_meta) + input_forcings = inputForcingMod[config_options_cfs.input_forcings[0]] + # TODO Can we construct this path from the config file? Parts of the path are currently hard-coded + # im time_handling.py, however + input_forcings.file_in2 = "config_data/CFSv2/cfs.20190923/06/6hrly_grib_01/flxf2019100106.01.2019092306.grb2" + input_forcings.fcst_hour2 = 192 + input_forcings.fcst_date1 = datetime.strptime("2019100106", "%Y%m%d%H%M") + input_forcings.fcst_date2 = input_forcings.fcst_date1 + + input_forcings.regrid_inputs(config_options_cfs,geo_meta,mpi_meta) + + # put a value of 1.5 at every gridpoint, for 10 mock variables + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 + + return input_forcings + +##### No bias correction tests ######### +@pytest.mark.mpi +def test_no_bias_correct(config_options, mpi_meta, unbiased_input_forcings): + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='no_bias_correct', equal=True) + + +##### NCAR table bias correction tests ######### +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_0z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_6z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 6 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_12z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 12 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_temp(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='T2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_rh(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='Q2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_u(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='U2D') + +@pytest.mark.mpi +def test_ncar_tbl_correction_18z_v(config_options, mpi_meta, unbiased_input_forcings): + unbiased_input_forcings.fcst_hour2 = 18 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_tbl_correction', var='V2D') + +##### NCAR blanket LW Bias Correction tests ######### +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_blanket_adjustment_lw_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_blanket_adjustment_lw', var='LWDOWN') + +##### NCAR HRRR SW bias correction tests ######### +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_0z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_6z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_12z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_18z(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_0z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_6z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_12z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +@pytest.mark.mpi +def test_ncar_sw_hrrr_bias_correct_18z_noana(config_options, geo_meta, mpi_meta, unbiased_input_forcings): + # This bias correction is affected by spatial position. Since the spatial point is different depending on how + # many processes we have, we skip this test for n_processes > 1 + if mpi_meta.size > 1: return + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=geo_meta, type='ncar_sw_hrrr_bias_correct', var='SWDOWN') + +##### NCAR HRRR temperature bias correction tests ######### +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_hrrr_bias_correct_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_hrrr_bias_correct', var='T2D') + +##### NCAR GFS temperature bias correction tests ######### +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +@pytest.mark.mpi +def test_ncar_temp_gfs_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_temp_gfs_bias_correct', var='T2D') + +##### NCAR GFS LW bias correction tests ######### +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +@pytest.mark.mpi +def test_ncar_lwdown_gfs_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_lwdown_gfs_bias_correct', var='LWDOWN') + +##### NCAR HRRR wind speed bias correction tests ######### +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_0z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_6z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_12z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_18z(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 1 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_0z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-01-01 00:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_6z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-04-01 06:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_12z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-07-01 12:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_hrrr_bias_correct_18z_noana(config_options, mpi_meta, unbiased_input_forcings): + config_options.b_date_proc = date('2023-10-01 18:00:00') + config_options.ana_flag = 0 + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_hrrr_bias_correct', var='U2D') + +##### NCAR GFS Wind Speed bias correction tests ######### +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_0z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-01-01 00:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_6z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-04-01 06:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_12z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-07-01 12:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +@pytest.mark.mpi +def test_ncar_wspd_gfs_bias_correct_18z(config_options, unbiased_input_forcings, mpi_meta): + config_options.b_date_proc = date('2023-10-01 18:00:00') + run_unit_test(config_options, mpi_meta, unbiased_input_forcings, type='ncar_wspd_gfs_bias_correct', var='U2D') + +##### CFS bias correction tests ######### +@pytest.mark.mpi +def test_cfsv2_nldas_nwm_bias_correct(config_options_cfs, unbiased_input_forcings_cfs, mpi_meta,geo_meta): + # Don't run for mpi processes > 0, since the + # CFS and NLDAS parameters are spatially dependent and we're not sure + # how we are splitting up the grid. + if mpi_meta.size > 1: + return + for var in var_list: + val = var_list[var] + unbiased_input_forcings_cfs.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + unbiased_input_forcings_cfs.coarse_input_forcings1 = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + unbiased_input_forcings_cfs.coarse_input_forcings2 = np.full((10,geo_meta.ny_local, geo_meta.nx_local),val) + run_unit_test(config_options_cfs, mpi_meta, unbiased_input_forcings_cfs, type='cfsv2_nldas_nwm_bias_correct', var=var) + +##### Tests for the top-level 'run_bias_correction()' function ####### +@pytest.mark.mpi +def test_run_bias_correction1(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.t2dBiasCorrectOpt = BiasCorrTempEnum.HRRR.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_temp_hrrr_bias_correct", varname='T2D') + +@pytest.mark.mpi +def test_run_bias_correction2(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.windBiasCorrectOpt = BiasCorrWindEnum.HRRR.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_wspd_hrrr_bias_correct", varname='U2D') + +@pytest.mark.mpi +def test_run_bias_correction3(unbiased_input_forcings, config_options, geo_meta, mpi_meta): + unbiased_input_forcings.q2dBiasCorrectOpt = BiasCorrHumidEnum.CUSTOM.name + + run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, "ncar_tbl_correction", varname='Q2D') + + +############ Helper functions ################ + +def run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta=None, + type='ncar_sw_hrrr_bias_correct', var='SWDOWN', equal=False): + for spec in expected_values[type]: + if 'fcst_hour' in spec and spec['fcst_hour'] != unbiased_input_forcings.fcst_hour2: + continue + if 'var' in spec and spec['var'] != var: + continue + if 'ana' in spec and spec['ana'] != config_options.ana_flag: + continue + if 'date' in spec and spec['date'] != config_options.b_date_proc: + continue + + funct = getattr(BC, type) + + run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=geo_meta, + compval=spec['value'], tolerance=spec['tolerance'], index=spec['index'], + var=var, equal=equal) + +def run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_meta, specname, varname="T2D"): + config_options.current_output_step = 60 + config_options.current_output_date = config_options.b_date_proc + + final_forcings = np.copy(unbiased_input_forcings.final_forcings) + + BC.run_bias_correction(unbiased_input_forcings,config_options,geo_meta,mpi_meta) + assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) + for spec in expected_values[specname]: + if 'fcst_hour' in spec and spec['fcst_hour'] != unbiased_input_forcings.fcst_hour2: + continue + if 'var' in spec and spec['var'] != varname: + continue + if 'ana' in spec and spec['ana'] != config_options.ana_flag: + continue + if 'date' in spec and spec['date'] != config_options.b_date_proc: + continue + + outId = getOutputId(config_options,unbiased_input_forcings, varname) + index = (outId, spec['index'][0], spec['index'][1]) + pointval = unbiased_input_forcings.final_forcings[index] + assert isWithinTolerance(pointval, spec['value'], spec['tolerance']) + +def run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=None, + compval=None, tolerance=None, equal=False, index=(0,0), var='T2D'): + """ + Run the provided bias_correction function, and sample the output to compare + against the expected value. Also ensure the overall original and bias_corrected grid sums are different. + :param funct: The function to call + :param config_options: + :param unbiased_input_forcings: + :param mpi_meta: + :param geo_meta: Can be None if not needed for function call + :param compval: Expected corrected value. Can be None to skip this test + :tolerance: Ensure that abs(compval-sampledval) <= tolerance. If None, the values must match exactly + :param equal: If true, the bias_corrected grid should be equivalent to the original grid + """ + #config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + unbiased_input_forcings.define_product() + final_forcings = np.copy(unbiased_input_forcings.final_forcings) + varnum = unbiased_input_forcings.input_map_output.index(var) + if geo_meta: + funct(unbiased_input_forcings, geo_meta, config_options, mpi_meta, varnum) + else: + funct(unbiased_input_forcings, config_options, mpi_meta, varnum) + + if equal: + assert np.sum(final_forcings) == np.sum(unbiased_input_forcings.final_forcings) + else: + assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) + + if compval is not None: + outId = getOutputId(config_options, unbiased_input_forcings, var) + idx = (outId, index[0], index[1]) + + pointval = unbiased_input_forcings.final_forcings[idx] + # TODO set acceptable tolerance + assert isWithinTolerance(pointval, compval, tolerance) + +def getOutputId(config_options, input_forcings, varname): + OutputEnum = config_options.OutputEnum + varnum = input_forcings.input_map_output.index(varname) + outId = 0 + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[varnum]: + outId = ind + break + + return outId + +def isWithinTolerance(val1, val2, tolerance=None): + """ + Make sure val1==val2, within the provided tolerance + """ + if tolerance is None: + tolerance = 0.0 + + diff = abs(val1 - val2) + return diff <= tolerance diff --git a/core/tests/test_config_options.py b/core/tests/test_config_options.py new file mode 100644 index 0000000..7ec8aab --- /dev/null +++ b/core/tests/test_config_options.py @@ -0,0 +1,48 @@ +import pytest +from core.config import ConfigOptions + +@pytest.mark.parametrize('invalid_config', [ + # Invalid values or types + 'yaml/configOptions_invalid1.yaml' + # Add more invalid configurations +]) +def test_read_config_invalid(invalid_config): + with pytest.raises(Exception): + config_options = ConfigOptions(invalid_config) + config_options.read_config() + +@pytest.mark.parametrize('missing_key', [ + # Missing required keys + 'Forecast', + 'Output' + # Add more missing keys +]) +def test_read_config_missing(missing_key): + config = 'yaml/configOptions_missing.yaml' + with pytest.raises(Exception): + config_options = ConfigOptions(config) + config_options.read_config() + +@pytest.mark.parametrize('empty_value', [ + # Empty values + 'yaml/configOptions_empty.yaml' + # Add more empty values +]) +def test_read_config_empty(empty_value): + with pytest.raises(Exception): + config_options = ConfigOptions(empty_value) + config_options.read_config() + +@pytest.fixture +def config_options(): + config_path = 'yaml/configOptions_valid.yaml' + return ConfigOptions(config_path) + +def test_read_config_valid(config_options): + config_options.read_config() + + # Assert expected values for valid configuration + assert config_options.output_dir == 'output/Hawaii_AnA_withMRMS_test' + assert config_options.fcst_freq == 60 + + # Add more assertions for other attributes diff --git a/core/tests/test_geo_meta.py b/core/tests/test_geo_meta.py new file mode 100644 index 0000000..b4adecf --- /dev/null +++ b/core/tests/test_geo_meta.py @@ -0,0 +1,37 @@ +import pytest +from netCDF4 import Dataset + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro + +from fixtures import * + +import numpy as np + + +@pytest.mark.mpi +def test_get_processor_bounds(geo_meta,mpi_meta): + geo_meta.get_processor_bounds() + + assert geo_meta.x_lower_bound == 0 + assert geo_meta.x_upper_bound == geo_meta.nx_global + + assert abs((geo_meta.ny_global / mpi_meta.size) - geo_meta.ny_local) < mpi_meta.size + assert abs(geo_meta.y_lower_bound - (mpi_meta.rank * geo_meta.ny_local)) < mpi_meta.size + assert abs(geo_meta.y_upper_bound - ((mpi_meta.rank + 1) * geo_meta.ny_local)) < mpi_meta.size + +@pytest.mark.mpi +def test_calc_slope(geo_meta,config_options): + config_options.read_config() + + idTmp = Dataset(config_options.geogrid,'r') + (slopeOut,slp_azi) = geo_meta.calc_slope(idTmp,config_options) + + assert slopeOut.shape == slp_azi.shape + assert slopeOut.size == slp_azi.size + + # test the sums of these grids are as expected, + # within floating point rounding tolerance + assert abs(np.sum(slopeOut) - 1527.7666) < 0.001 + assert abs(np.sum(slp_azi) - 58115.258) < 0.001 \ No newline at end of file diff --git a/core/tests/test_input_forcings.py b/core/tests/test_input_forcings.py new file mode 100644 index 0000000..8eb0aa2 --- /dev/null +++ b/core/tests/test_input_forcings.py @@ -0,0 +1,62 @@ +import pytest +from core.config import ConfigOptions +from core.forcingInputMod import input_forcings + + +@pytest.fixture +def config_options(): + config_path = './yaml/configOptions_valid.yaml' + yield ConfigOptions(config_path) + +@pytest.fixture +def input_forcings_sample(config_options): + config_options.read_config() + force_key = config_options.input_forcings[0] + InputDict = {} + InputDict[force_key] = input_forcings() + InputDict[force_key].keyValue = force_key + yield InputDict + +def test_define_product(config_options, input_forcings_sample): + force_key = config_options.input_forcings[0] + input_forcings_sample[force_key].forcingInputModYaml = "./yaml/inputForcings_valid.yaml" + input_forcings_sample[force_key].define_product() + # Assert expected values for valid configuration + assert input_forcings_sample[force_key].cycleFreq == 360 + assert input_forcings_sample[force_key].input_map_output == ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + + # Add more assertions for other attributes + +@pytest.mark.parametrize('invalid_config', ['./yaml/inputForcings_invalid.yaml' + # Add more invalid configurations +]) + +def test_define_config_invalid(invalid_config): + with pytest.raises(Exception): + + force_key = config_options.input_forcings[0] + input_forcings_sample[force_key].forcingInputModYaml = invalid_config + input_forcings_sample[force_key].define_product() + +@pytest.mark.parametrize('missing_key', [ + # Missing required keys + 'GFS_GLOBAL', + 'HRRR', + # Add more missing keys +]) +def test_define_config_missing(missing_key): + config = './yaml/inputForcings_missing.yaml' + with pytest.raises(Exception): + config_options = ConfigOptions(config) + config_options.read_config() + +@pytest.mark.parametrize('empty_value', [ + # Empty values + './yaml/inputForcings_empty.yaml' + # Add more empty values +]) +def test_define_config_empty(empty_value): + with pytest.raises(Exception): + config_options = ConfigOptions(empty_value) + config_options.read_config() + diff --git a/core/tests/test_mpi_meta.py b/core/tests/test_mpi_meta.py new file mode 100644 index 0000000..58123dc --- /dev/null +++ b/core/tests/test_mpi_meta.py @@ -0,0 +1,62 @@ +import pytest +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro +import numpy as np + +from fixtures import * + + + +@pytest.mark.mpi +def test_create_mpi_config(config_options, mpi_meta): + config_options.read_config() + +@pytest.mark.mpi +def test_broadcast_parameter(config_options, mpi_meta): + + # int + value = 999 if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == 999) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=int) + assert param == 999 + + # float + value = 999.999 if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == 999.999) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=float) + assert param == 999.999 + + # bool + value = True if mpi_meta.rank == 0 else None + assert (mpi_meta.rank == 0 and value == True) or (mpi_meta.rank != 0 and value is None) + param = mpi_meta.broadcast_parameter(value, config_options, param_type=bool) + assert param == True + + +@pytest.mark.mpi +def test_scatter_array(config_options, mpi_meta, geo_meta): + + target_data = None + if mpi_meta.rank == 0: + target_data = np.random.randn(geo_meta.ny_global,geo_meta.nx_global) + + # get the sum of the full grid. + value = np.sum(target_data) if mpi_meta.rank == 0 else None + value = mpi_meta.broadcast_parameter(value, config_options, param_type=float) + + target_data = mpi_meta.scatter_array(geo_meta,target_data,config_options) + + # None of the sub grids' sum should equal the sum of the full grid + # unless there's just a single process + assert np.sum(target_data) != value or mpi_meta.size == 1 + + # The x dimension size should be the same as the full grid + assert target_data.shape[1] == geo_meta.nx_global + + # The length of the y dimension of the sub grid should equal (full grid.y / num_processes), + # account for rounding when scattering the array + assert abs((target_data.shape[0] * mpi_meta.size) - geo_meta.ny_global) < mpi_meta.size + + + \ No newline at end of file diff --git a/core/tests/yaml/configOptions_cfs.yaml b/core/tests/yaml/configOptions_cfs.yaml new file mode 100644 index 0000000..5b93ed9 --- /dev/null +++ b/core/tests/yaml/configOptions_cfs.yaml @@ -0,0 +1,69 @@ +Forecast: + AnAFlag: false + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '201909230600' + RefcstEDateProc: '201910010600' + Shift: 0 + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: CFS_V2 + Longwave: CFS_V2 + Precip: CFS_V2 + Pressure: CFS_V2 + Shortwave: CFS_V2 + Temperature: CFS_V2 + Wind: CFS_V2 + Dir: config_data/CFSv2 + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: config_data/CFSv2 + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: CFS_V2 + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + Ensembles: + cfsEnsNumber: 1 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_empty.yaml b/core/tests/yaml/configOptions_empty.yaml new file mode 100644 index 0000000..900ec24 --- /dev/null +++ b/core/tests/yaml/configOptions_empty.yaml @@ -0,0 +1,67 @@ +Forecast: + AnAFlag: + Frequency: + LookBack: + RefcstBDateProc: + RefcstEDateProc: + Shift: + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_invalid1.yaml b/core/tests/yaml/configOptions_invalid1.yaml new file mode 100644 index 0000000..382eef2 --- /dev/null +++ b/core/tests/yaml/configOptions_invalid1.yaml @@ -0,0 +1,67 @@ +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: invalid + RefcstBDateProc: '2023022' + RefcstEDateProc: '202302261800000' + Shift: 0 + +Geospatial: + GeogridIn: config_data//DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_invalid2.yaml b/core/tests/yaml/configOptions_invalid2.yaml new file mode 100644 index 0000000..2690d87 --- /dev/null +++ b/core/tests/yaml/configOptions_invalid2.yaml @@ -0,0 +1,67 @@ +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '202302261200' + RefcstEDateProc: '202302261800' + Shift: 0 + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_missing.yaml b/core/tests/yaml/configOptions_missing.yaml new file mode 100644 index 0000000..074f706 --- /dev/null +++ b/core/tests/yaml/configOptions_missing.yaml @@ -0,0 +1,52 @@ +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_missingKey.yaml b/core/tests/yaml/configOptions_missingKey.yaml new file mode 100644 index 0000000..72c3687 --- /dev/null +++ b/core/tests/yaml/configOptions_missingKey.yaml @@ -0,0 +1,59 @@ +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/configOptions_valid.yaml b/core/tests/yaml/configOptions_valid.yaml new file mode 100644 index 0000000..2690d87 --- /dev/null +++ b/core/tests/yaml/configOptions_valid.yaml @@ -0,0 +1,67 @@ +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '202302261200' + RefcstEDateProc: '202302261800' + Shift: 0 + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: NONE + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml' diff --git a/core/tests/yaml/forcingInputMod.yaml b/core/tests/yaml/forcingInputMod.yaml new file mode 100644 index 0000000..4b7c081 --- /dev/null +++ b/core/tests/yaml/forcingInputMod.yaml @@ -0,0 +1,255 @@ +NLDAS: + product_name : NLDAS2_GRIB1 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF','DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +GFS_GLOBAL: + product_name : GFS_Production_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in : ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +HRRR: + product_name : HRRR_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/core/tests/yaml/inputForcings_empty.yaml b/core/tests/yaml/inputForcings_empty.yaml new file mode 100644 index 0000000..b7dc714 --- /dev/null +++ b/core/tests/yaml/inputForcings_empty.yaml @@ -0,0 +1,255 @@ +NLDAS: + product_name : + cycle_freq_minutes: + grib_vars_in: + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +GFS_GLOBAL: + product_name : GFS_Production_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in : ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +HRRR: + product_name : HRRR_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/core/tests/yaml/inputForcings_invalid.yaml b/core/tests/yaml/inputForcings_invalid.yaml new file mode 100644 index 0000000..bfefb0e --- /dev/null +++ b/core/tests/yaml/inputForcings_invalid.yaml @@ -0,0 +1,255 @@ +NLDAS: + product_name : 0 + cycle_freq_minutes: -60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF','DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: invalid + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +GFS_GLOBAL: + product_name : GFS_Production_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in : ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +HRRR: + product_name : HRRR_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/core/tests/yaml/inputForcings_missing.yaml b/core/tests/yaml/inputForcings_missing.yaml new file mode 100644 index 0000000..db2185e --- /dev/null +++ b/core/tests/yaml/inputForcings_missing.yaml @@ -0,0 +1,227 @@ +NLDAS: + product_name : NLDAS2_GRIB1 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF','DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/core/tests/yaml/inputForcings_valid.yaml b/core/tests/yaml/inputForcings_valid.yaml new file mode 100644 index 0000000..4b7c081 --- /dev/null +++ b/core/tests/yaml/inputForcings_valid.yaml @@ -0,0 +1,255 @@ +NLDAS: + product_name : NLDAS2_GRIB1 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF','DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NARR: + product_name : NARR_GRIB1 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +GFS_GLOBAL: + product_name : GFS_Production_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in : ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_CONUS: + product_name : NAM_Conus_Nest_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: None + grib_message_idx: None + input_map_to_outputs: None + +HRRR: + product_name : HRRR_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +RAP: + product_name : RAP_Conus_GRIB2 + cycle_freq_minutes: 60 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CFS_V2: + product_name : CFSv2_6Hr_Global_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +WRF_NEST_HI: + product_name : WRF_ARW_Hawaii_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in : ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','PSFC'] + +GFS_GLOBAL_25: + product_name : GFS_Production_025d_GRIB2 + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: [33,34,39,40,43,88,91,6] + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_1: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_2: + product_name : Custom_NetCDF_Hourly + cycle_freq_minutes: -9999 + grib_vars_in: None + grib_levels_in: None + netcdf_variables: ['T2D', 'Q2D', 'U10', 'V10', 'RAINRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +CUSTOM_3: + product_name : AORC + cycle_freq_minutes: -9999 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', + 'DSWRF', 'DLWRF', 'PRES'] + grib_levels_in: None + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI: + product_name : NAM_Nest_3km_Hawaii + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_PR: + product_name : NAM_Nest_3km_PuertoRico + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_AK: + product_name : NAM_Nest_3km_Alaska + cycle_freq_minutes: 360 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'PRATE', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'PRATE_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +NAM_NEST_HI_RAD: + product_name : NAM_Nest_3km_Hawaii_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +NAM_NEST_PR_RAD: + product_name : NAM_Nest_3km_PuertoRico_Radiation-Only + cycle_freq_minutes: 360 + grib_vars_in: ['DSWRF', 'DLWRF'] + grib_levels_in: ['surface', 'surface'] + netcdf_variables: ['DSWRF_surface', 'DLWRF_surface'] + grib_message_idx: None + input_map_to_outputs: ['SWDOWN','LWDOWN'] + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_GRIB2 + cycle_freq_minutes: 1440 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'PRES'] + grib_levels_in: ['80 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface'] + netcdf_variables: ['TMP_80maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D', 'Q2D', 'U2D', 'V2D', 'RAINRATE', 'PSFC'] + +HRRR_AK: + product_name : HRRR_Alaska_GRIB2 + cycle_freq_minutes: 180 + grib_vars_in: ['TMP', 'SPFH', 'UGRD', 'VGRD', 'APCP', 'DSWRF', + 'DLWRF', 'PRES'] + grib_levels_in: ['2 m above ground', '2 m above ground', + '10 m above ground', '10 m above ground', + 'surface', 'surface', 'surface', 'surface'] + netcdf_variables: ['TMP_2maboveground', 'SPFH_2maboveground', + 'UGRD_10maboveground', 'VGRD_10maboveground', + 'APCP_surface', 'DSWRF_surface', 'DLWRF_surface', + 'PRES_surface'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] + +HRRR_AK_EXT: + product_name : Alaska_ExtAnA + cycle_freq_minutes: 60 + grib_vars_in: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_levels_in: None + netcdf_variables: ['U2D', 'V2D', 'LWDOWN', 'RAINRATE', 'T2D', + 'Q2D', 'PSFC', 'SWDOWN'] + grib_message_idx: None + input_map_to_outputs: ['T2D','Q2D','U2D','V2D','RAINRATE','SWDOWN','LWDOWN','PSFC'] diff --git a/core/tests/yaml/outputVarAttr.yaml b/core/tests/yaml/outputVarAttr.yaml new file mode 100644 index 0000000..6a038be --- /dev/null +++ b/core/tests/yaml/outputVarAttr.yaml @@ -0,0 +1,89 @@ +U2D : + ind : 0 + units : 'm s-1' + standard_name : 'x_wind' + long_name : '10-m U-component of wind' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +V2D : + ind : 1 + units : 'm s-1' + standard_name : 'y_wind' + long_name : '10-m V-component of wind' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +LWDOWN : + ind : 2 + units : 'W m-2' + standard_name : 'surface_downward_longwave_flux' + long_name : 'Surface downward long-wave radiation flux' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +RAINRATE : + ind : 3 + units : 'mm s^-1' + standard_name : 'precipitation_flux' + long_name : 'Surface Precipitation Rate' + cell_methods : 'time: MEAN' + scale_factor : 1.0 + add_offset : 0.0 + least_significant_digit : 0 + +T2D : + ind : 4 + units : 'K' + standard_name : 'air_temperature' + long_name : '2-m Air Temperature' + cell_methods : 'time: point' + scale_factor : 0.01 + add_offset : 100.0 + least_significant_digit : 2 + +Q2D : + ind : 5 + units : 'kg kg-1' + standard_name : 'surface_specific_humidity' + long_name : '2-m Specific Humidity' + cell_methods : 'time: point' + scale_factor : 0.000001 + add_offset : 0.0 + least_significant_digit : 6 + +PSFC : + ind : 6 + units : 'Pa' + standard_name : 'air_pressure' + long_name : 'Surface Pressure' + cell_methods : 'time: point' + scale_factor : 0.1 + add_offset : 0.0 + least_significant_digit : 1 + +SWDOWN : + ind : 7 + units : 'W m-2' + standard_name : 'surface_downward_shortwave_flux' + long_name : 'Surface downward short-wave radiation flux' + cell_methods : 'time: point' + scale_factor : 0.001 + add_offset : 0.0 + least_significant_digit : 3 + +LQFRAC : + ind : 8 + units : '%' + standard_name : 'liquid_water_fraction' + long_name : 'Fraction of precipitation that is liquid vs. frozen' + cell_methods : 'time: point' + scale_factor : 0.1 + add_offset : 0.0 + least_significant_digit : 3 diff --git a/core/tests/yaml/suppPrecipMod.yaml b/core/tests/yaml/suppPrecipMod.yaml new file mode 100644 index 0000000..34bb47a --- /dev/null +++ b/core/tests/yaml/suppPrecipMod.yaml @@ -0,0 +1,87 @@ +MRMS: + product_name : MRMS_1HR_Radar_Only + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['RadarOnlyQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : ['RadarQualityIndex_0mabovemeansealevel'] + output_variables : RAINRATE + +MRMS_GAGE: + product_name : MRMS_1HR_Gage_Corrected + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['GaugeCorrQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : ['RadarQualityIndex_0mabovemeansealevel'] + output_variables : RAINRATE + +WRF_ARW_HI: + product_name : WRF_ARW_Hawaii_2p5km_PCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +WRF_ARW_PR: + product_name : WRF_ARW_PuertoRico_2p5km_PCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_CONUS_MS: + product_name : CONUS_MRMS_1HR_MultiSensor + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_HI_MS: + product_name : Hawaii_MRMS_1HR_MultiSensor + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +MRMS_SBCV2: + product_name : MRMS_LiquidWaterFraction + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['sbcv2_lwf'] + netcdf_rqi_variables : None + output_variables : LQFRAC + +AK_OPT1: + product_name : NBM_CORE_CONUS_APCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_OPT2: + product_name : NBM_CORE_ALASKA_APCP + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['APCP_surface'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_MRMS: + product_name : AK_MRMS + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : ['MultiSensorQPE01H_0mabovemeansealevel'] + netcdf_rqi_variables : None + output_variables : RAINRATE + +AK_NWS_IV: + product_name : AK_Stage_IV_Precip-MRMS + grib_vars_in : None + grib_levels_in : ['BLAH'] + netcdf_variables : [] + netcdf_rqi_variables : None + output_variables : RAINRATE diff --git a/core/time_handling.py b/core/time_handling.py index f83f33a..52f314f 100755 --- a/core/time_handling.py +++ b/core/time_handling.py @@ -6,7 +6,7 @@ import os import numpy as np - +from strenum import StrEnum from core import err_handler from core.forcingInputMod import input_forcings NETCDF = input_forcings.NETCDF @@ -20,6 +20,7 @@ def calculate_lookback_window(config_options): :param config_options: Abstract class holding job information. :return: Updated abstract class with updated datetime variables. """ + ForcingEnum = config_options.ForcingEnum # First calculate the current time in UTC. if config_options.realtime_flag: d_current_utc = datetime.datetime.utcnow() @@ -47,18 +48,17 @@ def calculate_lookback_window(config_options): # beginning of the processing window. dt_tmp = d_current_utc - config_options.b_date_proc n_fcst_steps = math.floor((dt_tmp.days*1440+dt_tmp.seconds/60.0) / config_options.fcst_freq) + config_options.nFcsts = int(n_fcst_steps) + 1 + config_options.e_date_proc = config_options.b_date_proc + datetime.timedelta( + seconds=n_fcst_steps * config_options.fcst_freq * 60) + #Special case for HRRR AK when we don't need/want more than one forecast cycle - if 19 in config_options.input_forcings: + if str(ForcingEnum.HRRR_AK.name) in config_options.input_forcings: n_fcst_steps = 0 config_options.nFcsts = int(n_fcst_steps) + 1 config_options.e_date_proc = config_options.b_date_proc + datetime.timedelta(seconds=config_options.look_back*60-config_options.fcst_freq*60) return - config_options.nFcsts = int(n_fcst_steps) + 1 - config_options.e_date_proc = config_options.b_date_proc + datetime.timedelta( - seconds=n_fcst_steps * config_options.fcst_freq * 60) - - def find_nldas_neighbors(input_forcings, config_options, d_current, mpi_config): """ Function to calculate the previous and after hourly NLDAS files for use in processing. @@ -482,14 +482,14 @@ def find_conus_hrrr_neighbors(input_forcings, config_options, d_current, mpi_con # Calculate expected file paths. tmp_file1 = input_forcings.inDir + '/hrrr.' + current_hrrr_cycle.strftime( - '%Y%m%d') + "/conus/hrrr.t" + current_hrrr_cycle.strftime('%H') + 'z.wrfsfcf' + \ + '%Y%m%d') + "/hrrr.t" + current_hrrr_cycle.strftime('%H') + 'z.wrfsfcf' + \ str(prev_hrrr_forecast_hour).zfill(2) + input_forcings.file_ext if mpi_config.rank == 0: config_options.statusMsg = "Previous HRRR file being used: " + tmp_file1 err_handler.log_msg(config_options, mpi_config) tmp_file2 = input_forcings.inDir + '/hrrr.' + current_hrrr_cycle.strftime( - '%Y%m%d') + "/conus/hrrr.t" + current_hrrr_cycle.strftime('%H') + 'z.wrfsfcf' \ + '%Y%m%d') + "/hrrr.t" + current_hrrr_cycle.strftime('%H') + 'z.wrfsfcf' \ + str(next_hrrr_forecast_hour).zfill(2) + input_forcings.file_ext if mpi_config.rank == 0: if mpi_config.rank == 0: @@ -599,17 +599,17 @@ def find_ak_hrrr_neighbors(input_forcings, config_options, d_current, mpi_config current_hrrr_hour = (current_hrrr_cycle.hour % 3) + 3 current_hrrr_cycle -= datetime.timedelta(hours=current_hrrr_hour) - + if current_hrrr_cycle.hour % 6 == 0: hrrr_horizon = 48 else: hrrr_horizon = 18 # print(f"HRRR cycle is {current_hrrr_cycle}, hour is {current_hrrr_hour}") - + # adjust horizon for this cycle - max_hours = hrrr_horizon - current_hrrr_hour - config_options.actual_output_steps = max_hours + # max_hours = hrrr_horizon - current_hrrr_hour + # config_options.actual_output_steps = max_hours # Calculate the previous file to process. min_since_last_output = (current_hrrr_hour * 60) % 60 @@ -1139,11 +1139,13 @@ def find_nam_nest_neighbors(input_forcings, config_options, d_current, mpi_confi if prev_nam_nest_forecast_hour == 0: prev_nam_nest_forecast_hour = 1 - if input_forcings.keyValue == 13 or input_forcings.keyValue == 16: + ForcingEnum = config_options.ForcingEnum + + if input_forcings.keyValue == str(ForcingEnum.NAM_NEST_HI.name) or input_forcings.keyValue == str(ForcingEnum.NAM_NEST_HI_RAD.name): domain_string = "hawaiinest" - elif input_forcings.keyValue == 14 or input_forcings.keyValue == 17: + elif input_forcings.keyValue == str(ForcingEnum.NAM_NEST_PR.name) or input_forcings.keyValue == str(ForcingEnum.NAM_NEST_PR_RAD.name): domain_string = "priconest" - elif input_forcings.keyValue == 15: + elif input_forcings.keyValue == str(ForcingEnum.NAM_NEST_AK.name): domain_string = "alaskanest" else: domain_string = '' @@ -1503,9 +1505,10 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr supplemental_precip.pcp_date1 = prev_mrms_date #supplemental_precip.pcp_date2 = next_mrms_date supplemental_precip.pcp_date2 = prev_mrms_date - + + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum # Calculate expected file paths. - if supplemental_precip.keyValue == 1: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS.name): tmp_file1 = supplemental_precip.inDir + "/RadarOnly_QPE/" + \ "RadarOnly_QPE_01H_00.00_" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + \ @@ -1517,7 +1520,7 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr "-" + supplemental_precip.pcp_date2.strftime('%H') + \ "0000" + supplemental_precip.file_ext + ('.gz' if supplemental_precip.fileType != NETCDF else '') - elif supplemental_precip.keyValue == 2: + elif supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_GAGE.name): tmp_file1 = supplemental_precip.inDir + "/GaugeCorr_QPE/" + \ "GaugeCorr_QPE_01H_00.00_" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + \ @@ -1529,7 +1532,7 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr "-" + supplemental_precip.pcp_date2.strftime('%H') + \ "0000" + supplemental_precip.file_ext + ('.gz' if supplemental_precip.fileType != NETCDF else '') - elif supplemental_precip.keyValue == 5 or supplemental_precip.keyValue == 6: + elif supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_CONUS_MS.name) or supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_HI_MS.name): tmp_file1 = supplemental_precip.inDir + "/MultiSensor_QPE_01H_Pass1/" + \ "MRMS_MultiSensor_QPE_01H_Pass1_00.00_" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + \ @@ -1540,7 +1543,7 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr supplemental_precip.pcp_date2.strftime('%Y%m%d') + \ "-" + supplemental_precip.pcp_date2.strftime('%H') + \ "0000" + supplemental_precip.file_ext + ('.gz' if supplemental_precip.fileType != NETCDF else '') - elif supplemental_precip.keyValue == 10: + elif supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_MRMS.name): tmp_file1 = supplemental_precip.inDir + "/MultiSensor_QPE_01H_Pass1/" + \ "MultiSensor_QPE_01H_Pass1_00.00_" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + \ @@ -1555,7 +1558,7 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr tmp_file1 = tmp_file2 = "" # Compose the RQI paths. - if supplemental_precip.keyValue == 1 or supplemental_precip.keyValue == 2: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS.name) or supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_GAGE.name): tmp_rqi_file1 = supplemental_precip.inDir + "/RadarQualityIndex/" + \ "RadarQualityIndex_00.00_" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + \ @@ -1630,7 +1633,7 @@ def find_hourly_mrms_radar_neighbors(supplemental_precip, config_options, d_curr # Ensure we have the necessary new file if mpi_config.rank == 0: - if not os.path.isfile(supplemental_precip.file_in2) and (supplemental_precip.keyValue == 5 or supplemental_precip.keyValue == 6): + if not os.path.isfile(supplemental_precip.file_in2) and (supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_CONUS_MS.name) or supplemental_precip.keyValue == str(SuppForcingPcpEnum.MRMS_HI_MS.name)): config_options.statusMsg = "MRMS file {} not found, will attempt to use {} instead.".format( supplemental_precip.file_in2, supplemental_precip.file_in1) err_handler.log_warning(config_options, mpi_config) @@ -1690,10 +1693,12 @@ def find_hourly_wrf_arw_neighbors(supplemental_precip, config_options, d_current err_handler.log_msg(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) + ForcingEnum = config_options.ForcingEnum + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum # Apply some logic here to account for the ARW weirdness associated with the # odd forecast cycles. fcst_horizon = -1 - if supplemental_precip.keyValue == 3: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.WRF_ARW_HI.name): # Data only available for 00/12 UTC cycles. if current_arw_cycle.hour != 0 and current_arw_cycle.hour != 12: if mpi_config.rank == 0: @@ -1704,7 +1709,7 @@ def find_hourly_wrf_arw_neighbors(supplemental_precip, config_options, d_current supplemental_precip.file_in2 = None return fcst_horizon = 48 - if supplemental_precip.keyValue == 4: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.WRF_ARW_PR.name): # Data only available for 06/18 UTC cycles. cycle = current_arw_cycle if cycle.hour != 6 and cycle.hour != 18: @@ -1775,7 +1780,7 @@ def find_hourly_wrf_arw_neighbors(supplemental_precip, config_options, d_current # Calculate expected file paths. tmp_file1 = tmp_file2 = "(none)" - if supplemental_precip.keyValue == 3 or supplemental_precip.keyValue == 8: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.WRF_ARW_HI.name) or supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_OPT1.name) or supplemental_precip.keyValue == str(ForcingEnum.WRF_NEST_HI.name): tmp_file1 = supplemental_precip.inDir + '/hiresw.' + \ current_arw_cycle.strftime('%Y%m%d') + '/hiresw.t' + \ current_arw_cycle.strftime('%H') + 'z.arw_2p5km.f' + \ @@ -1784,7 +1789,7 @@ def find_hourly_wrf_arw_neighbors(supplemental_precip, config_options, d_current current_arw_cycle.strftime('%Y%m%d') + '/hiresw.t' + \ current_arw_cycle.strftime('%H') + 'z.arw_2p5km.f' + \ str(next_arw_forecast_hour).zfill(2) + '.hi' + supplemental_precip.file_ext - elif supplemental_precip.keyValue == 4 or supplemental_precip.keyValue == 18: + elif supplemental_precip.keyValue == str(SuppForcingPcpEnum.WRF_ARW_PR.name): #or supplemental_precip.keyValue == str(ForcingEnum.WRF_NEST_HI.name): tmp_file1 = supplemental_precip.inDir + '/hiresw.' + \ current_arw_cycle.strftime('%Y%m%d') + '/hiresw.t' + \ current_arw_cycle.strftime('%H') + 'z.arw_2p5km.f' + \ @@ -1956,6 +1961,11 @@ def _find_ak_ext_ana_precip_stage4(supplemental_precip, config_options, d_curren #prev_stage4_date = datetime.datetime.fromtimestamp(d_prev_epoch - d_prev_epoch%six_hr_sec) prev_stage4_date = datetime.datetime.fromtimestamp(d_current_epoch - d_current_epoch%six_hr_sec) next_stage4_date = prev_stage4_date + + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum + + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum + # Set the input file frequency to be six-hourly. supplemental_precip.input_frequency = 360.0 @@ -1969,13 +1979,15 @@ def _find_ak_ext_ana_precip_stage4(supplemental_precip, config_options, d_curren stage4_in_dir = None # Calculate expected file paths. + tmp_file_ext = ".grb2" if supplemental_precip.fileType == 'GRIB2' else ".grb2.nc" - if stage4_in_dir and supplemental_precip.keyValue == 11: + if stage4_in_dir and supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_NWS_IV.name): tmp_file1 = f"{stage4_in_dir}/st4_ak.{supplemental_precip.pcp_date1.strftime('%Y%m%d%H.06h')}{tmp_file_ext}" #if d_current_epoch%six_hr_sec == 0: # tmp_file2 = f"{stage4_in_dir}/st4_ak.{supplemental_precip.pcp_date2.strftime('%Y%m%d%H.06h')}{tmp_file_ext}" #else: tmp_file2 = f"{stage4_in_dir}/st4_ak.{supplemental_precip.pcp_date2.strftime('%Y%m%d%H.06h')}{tmp_file_ext}" + else: tmp_file1 = tmp_file2 = "" @@ -2027,7 +2039,7 @@ def _find_ak_ext_ana_precip_stage4(supplemental_precip, config_options, d_curren # errMod.check_program_status(ConfigOptions, MpiConfig) # Ensure we have the necessary new file - if not os.path.isfile(supplemental_precip.file_in2) and supplemental_precip.keyValue == 11: + if not os.path.isfile(supplemental_precip.file_in2) and supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_NWS_IV.name): if mpi_config.rank == 0: config_options.statusMsg = "Stage IV file {} not found, will attempt to use {} instead.".format( supplemental_precip.file_in2, supplemental_precip.file_in1) @@ -2058,6 +2070,10 @@ def _find_ak_ext_ana_precip_mrms(supplemental_precip, config_options, d_current, prev_mrms_date = d_current - datetime.timedelta(hours=1) next_mrms_date = prev_mrms_date + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum + + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum + # Set the input file frequency to be hourly. supplemental_precip.input_frequency = 60.0 @@ -2071,7 +2087,7 @@ def _find_ak_ext_ana_precip_mrms(supplemental_precip, config_options, d_current, mrms_in_dir = None # Calculate expected file paths. - if supplemental_precip.keyValue == 11: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_NWS_IV.name): tmp_file1 = mrms_in_dir + "/MultiSensor_QPE_01H_Pass1/" + \ supplemental_precip.pcp_date1.strftime('%Y%m%d') + "/" + \ "MRMS_MultiSensor_QPE_01H_Pass1_00.00_" + \ @@ -2135,7 +2151,7 @@ def _find_ak_ext_ana_precip_mrms(supplemental_precip, config_options, d_current, # errMod.check_program_status(ConfigOptions, MpiConfig) # Ensure we have the necessary new file - if not os.path.isfile(supplemental_precip.file_in2) and supplemental_precip.keyValue == 11: + if not os.path.isfile(supplemental_precip.file_in2) and supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_NWS_IV.name): if mpi_config.rank == 0: config_options.statusMsg = "MRMS file {} not found, will attempt to use {} instead.".format( supplemental_precip.file_in2, supplemental_precip.file_in1) @@ -2178,7 +2194,7 @@ def find_ak_ext_ana_precip_neighbors(supplemental_precip, config_options, d_curr _find_ak_ext_ana_precip_stage4(supplemental_precip, config_options, d_current, mpi_config) -def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_current, mpi_config): +def find_hourly_nbm_neighbors(supplemental_precip, config_options, d_current, mpi_config): """ Function to calculate the previous and next NBM/CORE/CONUS files. This will also calculate the neighboring radar quality index (RQI) files as well. @@ -2189,10 +2205,10 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren :return: """ nbm_out_freq = { - 36: 60, - 240: 360 + 36: 60, + 240: 360 } - + # First we need to find the nearest previous and next hour, which is # the previous/next NBM files we will be using. current_yr = d_current.year @@ -2200,11 +2216,15 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren current_day = d_current.day current_hr = d_current.hour current_min = d_current.minute - + # First find the current NBM forecast cycle that we are using. current_nbm_cycle = config_options.current_fcst_cycle - \ datetime.timedelta(seconds=supplemental_precip.userCycleOffset * 60.0) + # if Alaska SR, shift to previous cycle and add 3 hours to forecast ('f') if using all NBM precip + if config_options.fcst_freq == 180 and supplemental_precip.keyValue == 9: + current_nbm_cycle -= datetime.timedelta(hours=3) + # Calculate the current forecast hour within this NBM cycle. dt_tmp = d_current - current_nbm_cycle current_nbm_hour = int(dt_tmp.days*24) + int(dt_tmp.seconds/3600.0) @@ -2214,7 +2234,9 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren supplemental_precip.input_frequency = 60.0 else: supplemental_precip.input_frequency = 360.0 - + + SuppForcingPcpEnum = config_options.SuppForcingPcpEnum + # Calculate the previous file to process. min_since_last_output = (current_nbm_hour*60) % supplemental_precip.input_frequency if min_since_last_output == 0: @@ -2242,7 +2264,7 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren prev_nbm_forecast_hour = 1 # Calculate expected file paths. - if supplemental_precip.keyValue == 8: + if supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_OPT1.name): tmp_file1 = supplemental_precip.inDir + "/blend." + \ current_nbm_cycle.strftime('%Y%m%d') + \ "/" + current_nbm_cycle.strftime('%H') + \ @@ -2255,7 +2277,7 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren "/core/blend.t" + current_nbm_cycle.strftime('%H') + \ "z.core.f" + str(prev_nbm_forecast_hour).zfill(3) + ".co" \ + supplemental_precip.file_ext - elif supplemental_precip.keyValue == 9: + elif supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_OPT2.name): tmp_file1 = supplemental_precip.inDir + "/blend." + \ current_nbm_cycle.strftime('%Y%m%d') + \ "/" + current_nbm_cycle.strftime('%H') + \ @@ -2272,8 +2294,8 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren tmp_file1 = tmp_file2 = "" if mpi_config.rank == 0: - config_options.statusMsg = "Prev NBM supplemental file: " + tmp_file2 - err_handler.log_msg(config_options, mpi_config) + # config_options.statusMsg = "Prev NBM supplemental file: " + tmp_file2 + # err_handler.log_msg(config_options, mpi_config) config_options.statusMsg = "Next NBM supplemental file: " + tmp_file1 err_handler.log_msg(config_options, mpi_config) err_handler.check_program_status(config_options, mpi_config) @@ -2289,7 +2311,7 @@ def find_hourly_nbm_apcp_neighbors(supplemental_precip, config_options, d_curren # Ensure we have the necessary new file if mpi_config.rank == 0: - if not os.path.isfile(supplemental_precip.file_in2) and ((supplemental_precip.keyValue == 8) or (supplemental_precip.keyValue == 9)): + if not os.path.isfile(supplemental_precip.file_in2) and ((supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_OPT1.name)) or (supplemental_precip.keyValue == str(SuppForcingPcpEnum.AK_OPT2.name))): config_options.statusMsg = "NBM file {} not found, will attempt to use {} instead.".format( supplemental_precip.file_in2, supplemental_precip.file_in1) err_handler.log_warning(config_options, mpi_config) diff --git a/genForcing.py b/genForcing.py index 9d48735..cf5d4fd 100755 --- a/genForcing.py +++ b/genForcing.py @@ -1,8 +1,12 @@ import argparse import os -import ESMF - +# ESMF was renamed to esmpy in v8.4.0 +try: + import esmpy as ESMF +except ModuleNotFoundError: + import ESMF + from core import config from core import err_handler from core import forcingInputMod @@ -69,7 +73,6 @@ def main(): err_handler.err_out_screen(job_meta.errMsg) # ESMF.Manager(debug=True) - # Initialize our WRF-Hydro geospatial object, which contains # information about the modeling domain, local processor # grid boundaries, and ESMF grid objects/fields to be used @@ -93,10 +96,9 @@ def main(): "Local grid Must have x/y dimension size of 2." err_handler.err_out_screen_para(job_meta.errMsg, mpi_meta) err_handler.check_program_status(job_meta, mpi_meta) - # Initialize our output object, which includes local slabs from the output grid. try: - OutputObj = ioMod.OutputObj(WrfHydroGeoMeta) + OutputObj = ioMod.OutputObj(WrfHydroGeoMeta, job_meta) except Exception: err_handler.err_out_screen_para(job_meta, mpi_meta) err_handler.check_program_status(job_meta, mpi_meta) @@ -112,6 +114,7 @@ def main(): err_handler.err_out_screen_para(job_meta, mpi_meta) err_handler.check_program_status(job_meta, mpi_meta) + # If we have specified supplemental precipitation products, initialize # the supp class. if job_meta.number_supp_pcp > 0: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec5986a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +attrs +autopep8 +backcall +certifi +cftime +charset-normalizer +cycler +decorator +entrypoints +h5py +ipykernel +ipython +netcdf4 +numpy +pytest +pytest-mpi +pyaml +PyYAML +requests +StrEnum diff --git a/setup.py b/setup.py index 6eacdaa..fb1fef2 100755 --- a/setup.py +++ b/setup.py @@ -6,8 +6,8 @@ packages=['core'], url='', license='', - author='Ryan Cabell', - author_email='rcabell@ucar.edu', + author='Ishita Srivastava', + author_email='ishitas@ucar.edu', description='', - install_requires=['netCDF4', 'numpy', 'ESMPy', 'mpi4py'] + install_requires=['netCDF4', 'numpy', 'mpi4py','esmpy'] ) From 60a470e58700c07381f18fa1d57d0170f0867aad Mon Sep 17 00:00:00 2001 From: Andrew Gaydos Date: Wed, 1 Nov 2023 12:48:06 -0600 Subject: [PATCH 33/33] Added downscaling tests. Moved some reusable code from bias correction tests to shared fixtures.py file --- core/tests/fixtures.py | 90 +++++++++++ core/tests/test_bias_correction.py | 74 --------- core/tests/test_downscale.py | 159 +++++++++++++++++++ core/tests/yaml/configOptions_downscale.yaml | 67 ++++++++ 4 files changed, 316 insertions(+), 74 deletions(-) create mode 100644 core/tests/test_downscale.py create mode 100644 core/tests/yaml/configOptions_downscale.yaml diff --git a/core/tests/fixtures.py b/core/tests/fixtures.py index f87e21a..24bf497 100644 --- a/core/tests/fixtures.py +++ b/core/tests/fixtures.py @@ -3,6 +3,7 @@ from core.parallel import MpiConfig from core.config import ConfigOptions from core.geoMod import GeoMetaWrfHydro +from core.forcingInputMod import initDict import numpy as np @@ -40,4 +41,93 @@ def config_options_cfs(): config_options.read_config() yield config_options +@pytest.fixture +def config_options_downscale(): + config_path = './yaml/configOptions_downscale.yaml' + config_options = ConfigOptions(config_path) + config_options.read_config() + yield config_options + +@pytest.fixture +def unbiased_input_forcings(config_options,geo_meta): + """ + Prepare the mock input forcing grid + """ + config_options.read_config() + + inputForcingMod = initDict(config_options,geo_meta) + input_forcings = inputForcingMod[config_options.input_forcings[0]] + + # put a value of 1.5 at every gridpoint, for 10 mock variables + input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) + input_forcings.fcst_hour2 = 12 + + return input_forcings + +def isWithinTolerance(val1, val2, tolerance=None): + """ + Make sure val1==val2, within the provided tolerance + """ + if tolerance is None: + tolerance = 0.0 + + diff = abs(val1 - val2) + return diff <= tolerance + + +def run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=None, + compval=None, tolerance=None, equal=False, index=(0,0), var='T2D', isDownscaleFunct=False): + """ + Run the provided function, and sample the output to compare + against the expected value. Also ensure the overall original and modified grid sums are different. + :param funct: The function to call + :param config_options: + :param unbiased_input_forcings: + :param mpi_meta: + :param geo_meta: Can be None if not needed for function call + :param compval: Expected corrected value. Can be None to skip this test + :tolerance: Ensure that abs(compval-sampledval) <= tolerance. If None, the values must match exactly + :param equal: If true, the modified grid should be equivalent to the original grid + """ + #config_options.read_config() + config_options.current_output_date = config_options.b_date_proc + config_options.current_output_step = 60 + + unbiased_input_forcings.define_product() + final_forcings = np.copy(unbiased_input_forcings.final_forcings) + if isDownscaleFunct: + funct(unbiased_input_forcings, config_options, geo_meta, mpi_meta) + else: + varnum = unbiased_input_forcings.input_map_output.index(var) + if geo_meta: + funct(unbiased_input_forcings, geo_meta, config_options, mpi_meta, varnum) + else: + funct(unbiased_input_forcings, config_options, mpi_meta, varnum) + + if equal: + assert np.sum(final_forcings) == np.sum(unbiased_input_forcings.final_forcings) + else: + assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) + + if compval is not None: + outId = getOutputId(config_options, unbiased_input_forcings, var) + idx = (outId, index[0], index[1]) + + pointval = unbiased_input_forcings.final_forcings[idx] + # TODO set acceptable tolerance + print("expected value: %s, got: %s" % (compval, pointval)) + assert isWithinTolerance(pointval, compval, tolerance) + +def getOutputId(config_options, input_forcings, varname): + OutputEnum = config_options.OutputEnum + varnum = input_forcings.input_map_output.index(varname) + outId = 0 + for ind, xvar in enumerate(OutputEnum): + if xvar.name == input_forcings.input_map_output[varnum]: + outId = ind + break + + return outId + + diff --git a/core/tests/test_bias_correction.py b/core/tests/test_bias_correction.py index 232adba..2530a7f 100644 --- a/core/tests/test_bias_correction.py +++ b/core/tests/test_bias_correction.py @@ -131,22 +131,6 @@ def date(datestr): # ####################################################### -@pytest.fixture -def unbiased_input_forcings(config_options,geo_meta): - """ - Prepare the mock input forcing grid - """ - config_options.read_config() - - inputForcingMod = initDict(config_options,geo_meta) - input_forcings = inputForcingMod[config_options.input_forcings[0]] - - # put a value of 1.5 at every gridpoint, for 10 mock variables - input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1.5) - input_forcings.fcst_hour2 = 12 - - return input_forcings - @pytest.fixture def unbiased_input_forcings_cfs(config_options_cfs,geo_meta,mpi_meta): """ @@ -624,62 +608,4 @@ def run_bias_correction(unbiased_input_forcings, config_options, geo_meta, mpi_m pointval = unbiased_input_forcings.final_forcings[index] assert isWithinTolerance(pointval, spec['value'], spec['tolerance']) -def run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=None, - compval=None, tolerance=None, equal=False, index=(0,0), var='T2D'): - """ - Run the provided bias_correction function, and sample the output to compare - against the expected value. Also ensure the overall original and bias_corrected grid sums are different. - :param funct: The function to call - :param config_options: - :param unbiased_input_forcings: - :param mpi_meta: - :param geo_meta: Can be None if not needed for function call - :param compval: Expected corrected value. Can be None to skip this test - :tolerance: Ensure that abs(compval-sampledval) <= tolerance. If None, the values must match exactly - :param equal: If true, the bias_corrected grid should be equivalent to the original grid - """ - #config_options.read_config() - config_options.current_output_date = config_options.b_date_proc - config_options.current_output_step = 60 - unbiased_input_forcings.define_product() - final_forcings = np.copy(unbiased_input_forcings.final_forcings) - varnum = unbiased_input_forcings.input_map_output.index(var) - if geo_meta: - funct(unbiased_input_forcings, geo_meta, config_options, mpi_meta, varnum) - else: - funct(unbiased_input_forcings, config_options, mpi_meta, varnum) - - if equal: - assert np.sum(final_forcings) == np.sum(unbiased_input_forcings.final_forcings) - else: - assert np.sum(final_forcings) != np.sum(unbiased_input_forcings.final_forcings) - - if compval is not None: - outId = getOutputId(config_options, unbiased_input_forcings, var) - idx = (outId, index[0], index[1]) - - pointval = unbiased_input_forcings.final_forcings[idx] - # TODO set acceptable tolerance - assert isWithinTolerance(pointval, compval, tolerance) - -def getOutputId(config_options, input_forcings, varname): - OutputEnum = config_options.OutputEnum - varnum = input_forcings.input_map_output.index(varname) - outId = 0 - for ind, xvar in enumerate(OutputEnum): - if xvar.name == input_forcings.input_map_output[varnum]: - outId = ind - break - - return outId - -def isWithinTolerance(val1, val2, tolerance=None): - """ - Make sure val1==val2, within the provided tolerance - """ - if tolerance is None: - tolerance = 0.0 - - diff = abs(val1 - val2) - return diff <= tolerance diff --git a/core/tests/test_downscale.py b/core/tests/test_downscale.py new file mode 100644 index 0000000..414d700 --- /dev/null +++ b/core/tests/test_downscale.py @@ -0,0 +1,159 @@ +import pytest +import numpy as np +from datetime import datetime + +from core.parallel import MpiConfig +from core.config import ConfigOptions +from core.geoMod import GeoMetaWrfHydro +from core.forcingInputMod import * +from core.enumConfig import * + +from fixtures import * +import core.downscale as DS + + +expected_values = { + 'no_downscale': [ + { 'index': (0,0), 'value': 1.5, 'tolerance': None} + ], + 'simple_lapse': [ + { 'index': (0,0), 'value': 2.1489999890327454, 'tolerance': None} + ], + 'param_lapse': [ + { 'index': (0,0), 'value': 3.0, 'tolerance': None} + ], + 'pressure_down_classic': [ + { 'index': (0,0), 'value': 4.9140393659641175, 'tolerance': None} + ], + 'q2_down_classic': [ + { 'index': (0,0), 'value': 0.9992289527938468, 'tolerance': None} + ], + 'nwm_monthly_PRISM_downscale': [ + { 'index': (0,0), 'value': 2.357142857142857, 'tolerance': None} + ], + 'ncar_topo_adj': [ + { 'index': (0,0), 'value': 1400.0, 'tolerance': None} + ] +} + + +############### Downscale function tests ########################## + + +@pytest.mark.mpi +def test_no_downscale(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, type='no_downscale', equal=True) + +@pytest.mark.mpi +def test_simple_lapse(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + unbiased_input_forcings.q2dDownscaleOpt = DownScaleHumidEnum.REGRID_TEMP_PRESS + unbiased_input_forcings.height = geo_meta.height + 100 + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, type='simple_lapse') + +@pytest.mark.mpi +def test_param_lapse(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + unbiased_input_forcings.q2dDownscaleOpt = DownScaleHumidEnum.REGRID_TEMP_PRESS + unbiased_input_forcings.height = geo_meta.height + 100 + unbiased_input_forcings.paramDir = config_options_downscale.dScaleParamDirs[0] + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, type='param_lapse') + +@pytest.mark.mpi +def test_pressure_down_classic(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + unbiased_input_forcings.height = geo_meta.height + 100 + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, + type='pressure_down_classic',var='PSFC') + +@pytest.mark.mpi +def test_q2_down_classic(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + unbiased_input_forcings.t2dTmp = unbiased_input_forcings.t2dTmp * 0 + 273.15 + unbiased_input_forcings.psfcTmp = unbiased_input_forcings.psfcTmp * 0 + 100000 + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, + type='q2_down_classic',var='Q2D') + +@pytest.mark.mpi +def test_nwm_monthly_PRISM_downscale(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + config_options_downscale.current_output_date = datetime.strptime("20101001", "%Y%m%d") + config_options_downscale.prev_output_date = datetime.strptime("20100901", "%Y%m%d") + unbiased_input_forcings.paramDir = config_options_downscale.dScaleParamDirs[0] + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, + type='nwm_monthly_PRISM_downscale',var='RAINRATE') + +@pytest.mark.mpi +def test_ncar_topo_adj(unbiased_input_forcings, config_options_downscale, geo_meta, mpi_meta): + config_options_downscale.current_output_date = datetime.strptime("2010122000", "%Y%m%d%H") + geo_meta.latitude_grid = geo_meta.latitude_grid + 20.0 + unbiased_input_forcings.final_forcings = np.full((10,geo_meta.ny_local, geo_meta.nx_local),1500.0) + run_unit_test(config_options_downscale, mpi_meta, unbiased_input_forcings, geo_meta, + type='ncar_topo_adj',var='SWDOWN') + + +############### Helper function tests ########################## + + +@pytest.mark.mpi +def test_radconst(config_options_downscale): + config_options_downscale.current_output_date = datetime.strptime("20230427","%Y%m%d") + DECLIN, SOLCON = DS.radconst(config_options_downscale) + + assert isWithinTolerance(DECLIN, 0.23942772252574912, None) + assert isWithinTolerance(SOLCON, 1350.9227993143058, None) + + config_options_downscale.current_output_date = datetime.strptime("20231027","%Y%m%d") + DECLIN, SOLCON = DS.radconst(config_options_downscale) + + assert isWithinTolerance(DECLIN, -0.24225978751208468, None) + assert isWithinTolerance(SOLCON, 1388.352238489423, None) + + +@pytest.mark.mpi +def test_calc_coszen(config_options_downscale, geo_meta, mpi_meta): + # Skip this test for nprocs > 1, since the latitudes will be + # different for each scattered array + if mpi_meta.size > 1 and mpi_meta.rank != 0: + return True + + config_options_downscale.current_output_date = datetime.strptime("2023042706","%Y%m%d%H") + declin, solcon = DS.radconst(config_options_downscale) + coszen, hrang = DS.calc_coszen(config_options_downscale, declin, geo_meta) + + assert isWithinTolerance(coszen[0][0], -0.24323696, 1e-7) + assert isWithinTolerance(hrang[0][0], -4.3573823, 1e-7) + + config_options_downscale.current_output_date = datetime.strptime("2023102718","%Y%m%d%H") + declin, solcon = DS.radconst(config_options_downscale) + coszen, hrang = DS.calc_coszen(config_options_downscale, declin, geo_meta) + + assert isWithinTolerance(coszen[0][0], 0.29333305, 1e-7) + assert isWithinTolerance(hrang[0][0], -1.1556603, 1e-7) + + +@pytest.mark.mpi +def test_rel_hum(config_options_downscale, unbiased_input_forcings, geo_meta): + unbiased_input_forcings.t2dTmp = np.full((geo_meta.ny_local, geo_meta.nx_local),300) + unbiased_input_forcings.psfcTmp = np.full((geo_meta.ny_local, geo_meta.nx_local),100000) + unbiased_input_forcings.final_forcings = np.full((10, geo_meta.ny_local, geo_meta.nx_local),0.015) + + rh = DS.rel_hum(unbiased_input_forcings, config_options_downscale) + + assert isWithinTolerance(rh[0][0], 68.33107966724083, None) + + +@pytest.mark.mpi +def test_mixhum_ptrh(config_options_downscale, unbiased_input_forcings, geo_meta): + unbiased_input_forcings.t2dTmp = np.full((geo_meta.ny_local, geo_meta.nx_local),300) + unbiased_input_forcings.psfcTmp = np.full((geo_meta.ny_local, geo_meta.nx_local),100000) + unbiased_input_forcings.final_forcings = np.full((10, geo_meta.ny_local, geo_meta.nx_local),0.015) + + relHum = DS.rel_hum(unbiased_input_forcings, config_options_downscale) + + rh = DS.mixhum_ptrh(unbiased_input_forcings, relHum, 2, config_options_downscale) + + assert isWithinTolerance(rh[0][0], 9.039249311422024, None) + + +def run_unit_test(config_options, mpi_meta, unbiased_input_forcings, geo_meta, type=None, equal=False, var='T2D'): + funct = getattr(DS, type) + for spec in expected_values[type]: + run_and_compare(funct, config_options, unbiased_input_forcings, mpi_meta, geo_meta=geo_meta, equal=equal, + compval=spec['value'], tolerance=spec['tolerance'], index=spec['index'], var=var, isDownscaleFunct=True) + diff --git a/core/tests/yaml/configOptions_downscale.yaml b/core/tests/yaml/configOptions_downscale.yaml new file mode 100644 index 0000000..0cc3688 --- /dev/null +++ b/core/tests/yaml/configOptions_downscale.yaml @@ -0,0 +1,67 @@ +Forecast: + AnAFlag: true + Frequency: 60 + LookBack: 180 + RefcstBDateProc: '202302261200' + RefcstEDateProc: '202302261800' + Shift: 0 + +Geospatial: + GeogridIn: config_data/DOMAIN_Default/geo_em.d01.1km_Hawaii_NWMv3.0.nc + SpatialMetaIn: config_data/DOMAIN_Default/GEOGRID_LDASOUT_Spatial_Metadata_1km_Hawaii_NWMv3.0.nc + +Input: +- BiasCorrection: + Humidity: NONE + Longwave: NONE + Precip: NONE + Pressure: NONE + Shortwave: NONE + Temperature: NONE + Wind: NONE + Dir: config_data/NAM.Hawaii + Downscaling: + Humidity: REGRID_TEMP_PRESS + ParamDir: config_data/params + Precip: NONE + Pressure: ELEV + Shortwave: ELEV + Temperature: LAPSE_675 + Forcing: NAM_NEST_HI + Horizon: 60 + IgnoredBorderWidths: 0.0 + Mandatory: true + Offset: 0 + RegriddingOpt: ESMF_BILINEAR + TemporalInterp: NONE + Type: GRIB2 + +Output: + CompressOutput: false + Dir: output/Hawaii_AnA_withMRMS_test + FloatOutput: SCALE_OFFSET + Frequency: 60 + ScratchDir: output/Hawaii_AnA_withMRMS_test + +Regridding: + WeightsDir: null + +Retrospective: + Flag: false + +SuppForcing: +- Pcp: MRMS + PcpDir: config_data/HIRESW.Hawaii + PcpInputOffsets: 0 + PcpMandatory: false + PcpParamDir: config_data/Puerto_Rico + PcpTemporalInterp: 0 + PcpType: GRIB2 + RegridOptPcp: 1 + RqiMethod: NONE + RqiThreshold: 1.0 + +YamlConfig: + forcingInputModYaml: 'yaml/forcingInputMod.yaml' + outputVarAttrYaml: 'yaml/outputVarAttr.yaml' + suppPrecipModYaml: 'yaml/suppPrecipMod.yaml'