Refactor updates

.github/workflows/tests.yml

@@ -11,68 +11,33 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8  # Specify the desired Python version
-
-      - name: Set up Conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: actions/checkout@v3
         with:
-          channel: 'defaults'  # Specify the Conda channel you want to use
-          python-version: 3.8  # Specify the Python version for the Conda environment
-
-      - name: Set up Conda and initialize shell
+          path: WrfHydroForcing
+
+      - name: Run testing container
         run: |
-          conda init bash
-          conda create -n esmpy_env
-          eval "$(conda shell.bash hook)"
-          source activate esmpy_env
-        shell: bash
+          chmod -R 0777 $GITHUB_WORKSPACE/WrfHydroForcing && \
+          docker run -t --name test_container \
+           -v $GITHUB_WORKSPACE/WrfHydroForcing:/home/docker/WrfHydroForcing \
+           wrfhydro/wrf_hydro_forcing:latest 
 
-      - name: Set up ESMF and wgrib2
-        run: |
-          mkdir $HOME/.esmf-src \
-          cd $HOME/.esmf-src \
-          wget http://www.earthsystemmodeling.org/esmf_releases/public/ESMF_7_1_0r/esmf_7_1_0r_src.tar.gz \
-          tar xfz esmf_7_1_0r_src.tar.gz \
-          cd esmf \
-          export ESMF_DIR=$PWD \
-          export ESMF_COMM=mpich3 \
-          make \
-          make install
-
-          cd $HOME/.esmf-src/esmf/src/addon/ESMPy
-          python setup.py build --ESMFMKFILE=$HOME/.esmf-src/esmf/DEFAULTINSTALLDIR/lib/libO/Linux.gfortran.64.mpich3.default/esmf.mk install
+      # - name: Copy test results from container
+      #   if: ${{ always() }}
+      #   run: docker cp test_container:/home/docker/test_out $GITHUB_WORKSPACE/test_report
 
-          cd $HOME
-          wget ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz
-          tar xfz wgrib2.tgz
-          rm wgrib2.tgz
-          cd grib2
-          export CC=gcc
-          export FC=gfortran
-          make
-          make lib
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt  # If you have a requirements.txt file
-          conda install -c conda-forge esmpy
-          python setup.py install
 
-      - name: Run pytest
-        run: pytest
+      # - name: Run pytest
+      #   run: |
+      #     cd core/tests
+      #     pytest
 
-      # Optional: Generate test coverage report
-      - name: Install coverage tool
-        run: pip install coverage
+      # # Optional: Generate test coverage report
+      # - name: Install coverage tool
+      #   run: pip install coverage
 
-      - name: Run pytest with coverage
-        run: coverage run -m pytest
+      # - name: Run pytest with coverage
+      #   run: coverage run -m pytest
 
-      - name: Generate coverage report
-        run: coverage html -i
+      # - name: Generate coverage report
+      #   run: coverage html -i

core/Regridding/regrid_base.py

@@ -3,7 +3,12 @@
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 
-import ESMF
+# ESMF was renamed to esmpy in v8.4.0
+try:
+    import esmpy as ESMF
+except ModuleNotFoundError:
+    import ESMF
+
 import numpy as np
 
 from core import err_handler

core/bias_correction.py

@@ -44,6 +44,7 @@ def run_bias_correction(input_forcings, config_options, geo_meta_wrf_hydro, mpi_
         str(BiasCorrTempEnum.GFS.name)   : ncar_temp_gfs_bias_correct,
         str(BiasCorrTempEnum.HRRR.name)  : ncar_temp_hrrr_bias_correct
     }
+
     bias_correct_temperature[input_forcings.t2dBiasCorrectOpt](input_forcings, config_options, mpi_config, 0)
 
     err_handler.check_program_status(config_options, mpi_config)
@@ -383,7 +384,7 @@ def ncar_sw_hrrr_bias_correct(input_forcings, geo_meta_wrf_hydro, config_options
     # the cosine of the sol_zen_ang is proportional to the solar intensity
     # (not accounting for humidity or cloud cover)
     sol_cos = np.sin(geo_meta_wrf_hydro.latitude_grid * d2r) * math.sin(decl) + np.cos(geo_meta_wrf_hydro.latitude_grid * d2r) * math.cos(decl) * np.cos(ha)
-    
+
     # Check for any values outside of [-1,1] (this can happen due to floating point rounding)
     sol_cos[np.where(sol_cos < -1.0)] = -1.0
     sol_cos[np.where(sol_cos > 1.0)] = 1.0
@@ -794,6 +795,7 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force
     ugrd_idx = input_forcings.grib_vars.index('UGRD')
     vgrd_idx = input_forcings.grib_vars.index('VGRD')
 
+    OutputEnum = config_options.OutputEnum
     for ind, xvar in enumerate(OutputEnum):
             if xvar.name == input_forcings.input_map_output[ugrd_idx]:
                 u_in = ind
@@ -820,20 +822,19 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force
     # TODO: cache the "other" value so we don't repeat this calculation unnecessarily
 
     bias_corrected = ugrid_out if force_num == ugrd_idx else vgrid_out
-    OutputEnum = config_options.OutputEnum
     for ind, xvar in enumerate(OutputEnum):
             if xvar.name == input_forcings.input_map_output[force_num]:
                 outId = ind
                 break
     wind_in = None
     try:
-        wind_in = input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :]
+        wind_in = input_forcings.final_forcings[outId, :, :]
     except NumpyExceptions as npe:
         config_options.errMsg = "Unable to extract incoming windspeed from forcing object for: " + \
                                 input_forcings.productName + " (" + str(npe) + ")"
         err_handler.log_critical(config_options, mpi_config)
         err_handler.check_program_status(config_options, mpi_config)
-
+    
     ind_valid = None
     try:
         ind_valid = np.where(wind_in != config_options.globalNdv)
@@ -852,7 +853,7 @@ def ncar_wspd_gfs_bias_correct(input_forcings, config_options, mpi_config, force
         err_handler.check_program_status(config_options, mpi_config)
 
     try:
-        input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = wind_in[:, :]
+        input_forcings.final_forcings[outId, :, :] = wind_in[:, :]
     except NumpyExceptions as npe:
         config_options.errMsg = "Unable to place temporary windspeed array back into forcing object for: " + \
                                 input_forcings.productName + " (" + str(npe) + ")"
@@ -970,6 +971,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for
         # Open the NetCDF file.
         try:
             id_nldas_param = Dataset(nldas_param_file, 'r')
+            id_nldas_param.set_auto_mask(False)                 # don't mask missing since it won't survive broadcast
         except OSError as err:
             config_options.errMsg = "Unable to open parameter file: " + nldas_param_file + " (" + str(err) + ")"
             err_handler.log_critical(config_options, mpi_config)
@@ -1078,6 +1080,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for
         nldas_param_1 = None
         nldas_param_2 = None
         nldas_zero_pcp = None
+
     err_handler.check_program_status(config_options, mpi_config)
 
     # Scatter NLDAS parameters
@@ -1298,7 +1301,7 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for
     #     regridding object.
     # 6.) Place the data into the final output arrays for further processing (downscaling).
     # 7.) Reset variables for memory efficiency and exit the routine.
-    
+
     np.seterr(all='ignore')
 
     if mpi_config.rank == 0:
@@ -1314,12 +1317,15 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for
         config_options.statusMsg = "Looping over local arrays to calculate bias corrections."
         err_handler.log_msg(config_options, mpi_config)
     # Process each of the pixel cells for this local processor on the CFS grid.
+    outId = None
+    for ind, xvar in enumerate(config_options.OutputEnum):
+        if xvar.name == input_forcings.input_map_output[force_num]:
+            outId = ind
+            break
     for x_local in range(0, input_forcings.nx_local):
         for y_local in range(0, input_forcings.ny_local):
-            cfs_prev_tmp = input_forcings.coarse_input_forcings1[input_forcings.input_map_output[force_num],
-                                                                 y_local, x_local]
-            cfs_next_tmp = input_forcings.coarse_input_forcings2[input_forcings.input_map_output[force_num],
-                                                                 y_local, x_local]
+            cfs_prev_tmp = input_forcings.coarse_input_forcings1[outId, y_local, x_local]
+            cfs_next_tmp = input_forcings.coarse_input_forcings2[outId, y_local, x_local]
 
             # Check for any missing parameter values. If any missing values exist,
             # set this flag to False. Further down, if it's False, we will simply
@@ -1541,9 +1547,9 @@ def cfsv2_nldas_nwm_bias_correct(input_forcings, config_options, mpi_config, for
     err_handler.check_program_status(config_options, mpi_config)
 
     try:
-        input_forcings.final_forcings[input_forcings.input_map_output[force_num], :, :] = \
+        input_forcings.final_forcings[outId, :, :] = \
             input_forcings.esmf_field_out.data
     except NumpyExceptions as npe:
         config_options.errMsg = "Unable to extract ESMF field data for CFSv2: " + str(npe)
         err_handler.log_critical(config_options, mpi_config)
-    err_handler.check_program_status(config_options, mpi_config)
+    err_handler.check_program_status(config_options, mpi_config)

core/config.py

@@ -98,7 +98,7 @@ def __init__(self, config):
         self.customFcstFreq = None
         self.rqiMethod = None
         self.rqiThresh = 1.0
-        self.globalNdv = -9999.0
+        self.globalNdv = -999999.0
         self.d_program_init = datetime.datetime.utcnow()
         self.errFlag = 0
         self.nwmVersion = None
@@ -140,23 +140,43 @@ def read_config(self):
         self.number_inputs = len(inputs)
 
         # Create Enums dynamically from the yaml files
-        self.forcingInputModYaml = config['YamlConfig']['forcingInputModYaml']
+        try:
+            yaml_config = config['YamlConfig']
+        except KeyError:
+            raise KeyError("Unable to locate YamlConfig in configuration file.")
+            err_handler.err_out_screen('Unable to locate YamlConfig in configuration file.')
+
+        try:
+            self.forcingInputModYaml = yaml_config['forcingInputModYaml']
+        except KeyError:
+            raise KeyError("Unable to locate forcingInputModYaml in configuration file.")
+            err_handler.err_out_screen('Unable to locate forcingInputModYaml in configuration file.')
+
         forcing_yaml_stream = open(self.forcingInputModYaml)
         forcingConfig = yaml.safe_load(forcing_yaml_stream)
         dynamicForcing = {}
         for k in forcingConfig.keys():
             dynamicForcing[k] = k
         self.ForcingEnum = enum.Enum('ForcingEnum', dynamicForcing)
 
-        self.suppPrecipModYaml   = config['YamlConfig']['suppPrecipModYaml']
+        try:
+            self.suppPrecipModYaml   = yaml_config['suppPrecipModYaml']
+        except KeyError:
+            raise KeyError("Unable to locate suppPrecipModYaml in configuration file.")
+            err_handler.err_out_screen('Unable to locate suppPrecipModYaml in configuration file.')
+
         supp_yaml_stream = open(self.suppPrecipModYaml)
         suppConfig = yaml.safe_load(supp_yaml_stream)
         dynamicSupp = {}
         for k in suppConfig.keys():
             dynamicSupp[k] = k
         self.SuppForcingPcpEnum = enum.Enum('SuppForcingPcpEnum', dynamicSupp)
+        try:
+            self.outputVarAttrYaml  = yaml_config['outputVarAttrYaml']
+        except KeyError:
+            raise KeyError("Unable to locate outputVarAttrYaml in configuration file.")
+            err_handler.err_out_screen('Unable to locate outputVarAttrYaml in configuration file.')
 
-        self.outputVarAttrYaml  = config['YamlConfig']['outputVarAttrYaml']
         out_yaml_stream = open(self.outputVarAttrYaml)
         outConfig = yaml.safe_load(out_yaml_stream)
         dynamicOut = {}
@@ -401,25 +421,25 @@ def read_config(self):
         # Putting a constraint here that CFSv2-NLDAS bias correction (NWM only) is chosen, it must be turned on
         # for ALL variables.
         if self.runCfsNldasBiasCorrect:
-            if set(self.precipBiasCorrectOpt) != {'CFS_V2'}:
+            if self.precipBiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'Precipitation under this configuration.')
-            if min(self.lwBiasCorrectOpt) != {'CFS_V2'}:
+            if self.lwBiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'long-wave radiation under this configuration.')
-            if min(self.swBiasCorrectOpt) != {'CFS_V2'}:
+            if self.swBiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'short-wave radiation under this configuration.')
-            if min(self.t2BiasCorrectOpt) != {'CFS_V2'}:
+            if self.t2BiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'surface temperature under this configuration.')
-            if min(self.windBiasCorrect) != {'CFS_V2'}:
+            if self.windBiasCorrect != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'wind forcings under this configuration.')
-            if min(self.q2BiasCorrectOpt) != {'CFS_V2'}:
+            if self.q2BiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'specific humidity under this configuration.')
-            if min(self.psfcBiasCorrectOpt) != {'CFS_V2'}:
+            if self.psfcBiasCorrectOpt != ['CFS_V2']:
                 err_handler.err_out_screen('CFSv2-NLDAS NWM bias correction must be activated for '
                                            'surface pressure under this configuration.')
             # Make sure we don't have any other forcings activated. This can only be ran for CFSv2.
@@ -430,7 +450,7 @@ def read_config(self):
 
         # Read in the temperature downscaling options.
         # Create temporary array to hold flags of if we need input parameter files.
-        param_flag = np.empty([len(self.input_forcings)], np.int)
+        param_flag = np.empty([len(self.input_forcings)], int)
         param_flag[:] = 0
         try:
             self.t2dDownscaleOpt = [input['Downscaling']['Temperature'] for input in inputs]
@@ -944,7 +964,6 @@ def read_config(self):
             except KeyError:
                 err_handler.err_out_screen('Unable to locate SuppPcpDirectories in SuppForcing section '
                                            'in the configuration file.')
-
             # Loop through and ensure all supp pcp directories exist. Also strip out any whitespace
             # or new line characters.
             for dirTmp in range(0, len(self.supp_precip_dirs)):
@@ -1059,7 +1078,7 @@ def read_config(self):
         for optTmp in self.input_forcings:
             if optTmp == 'CFS_V2':
                 try:
-                    self.cfsv2EnsMember = input['Ensembles']['cfsEnsNumber']
+                    self.cfsv2EnsMember = [input['Ensembles']['cfsEnsNumber'] for input in inputs if 'cfsEnsNumber' in input['Ensembles']][0]
                 except KeyError:
                     err_handler.err_out_screen('Unable to locate cfsEnsNumber under the Ensembles '
                                                'section of the configuration file')
@@ -1072,4 +1091,4 @@ def use_data_at_current_time(self):
             hrs_since_start = self.current_output_date - self.current_fcst_cycle
             return hrs_since_start <= datetime.timedelta(hours = self.supp_pcp_max_hours)
         else:
-            return True 
+            return True

core/disaggregateMod.py

@@ -114,9 +114,9 @@ def ext_ana_disaggregate(input_forcings, supplemental_precip, config_options, mp
 
     ana_sum = np.array([],dtype=np.float32)
     target_data = np.array([],dtype=np.float32)
-    ana_all_zeros = np.array([],dtype=np.bool)
-    ana_no_zeros = np.array([],dtype=np.bool)
-    target_data_no_zeros = np.array([],dtype=np.bool)
+    ana_all_zeros = np.array([],dtype=bool)
+    ana_no_zeros = np.array([],dtype=bool)
+    target_data_no_zeros = np.array([],dtype=bool)
     if mpi_config.rank == 0:
         config_options.statusMsg = f"Performing hourly disaggregation of {supplemental_precip.file_in2}"
         err_handler.log_msg(config_options, mpi_config)

core/downscale.py

@@ -764,15 +764,17 @@ def TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro,input_forcings,ConfigOptions,COSZEN,declin
 
     COSZEN[np.where(COSZEN < 1E-4)] = 1E-4
 
-    corr_frac = np.empty([ny, nx], np.int)
-    # shadow_mask = np.empty([ny,nx],np.int)
-    diffuse_frac = np.empty([ny, nx], np.int)
+    corr_frac = np.empty([ny, nx], int)
+
+    # shadow_mask = np.empty([ny,nx],int)
+    diffuse_frac = np.empty([ny, nx], int)
     corr_frac[:, :] = 0
     diffuse_frac[:, :] = 0
     # shadow_mask[:,:] = 0
 
     indTmp = np.where((GeoMetaWrfHydro.slope[:,:] == 0.0) &
                       (SWDOWN <= 10.0))
+
     corr_frac[indTmp] = 1
 
     term1 = np.sin(xxlat) * np.cos(hrang2d)

core/forecastMod.py

@@ -123,7 +123,7 @@ def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMo
         show_message = True
         for outStep in range(1, ConfigOptions.num_output_steps + 1):
             # Reset out final grids to missing values.
-            OutputObj.output_local[:, :, :] = -9999.0
+            OutputObj.output_local[:, :, :] = ConfigOptions.globalNdv
 
             ConfigOptions.current_output_step = outStep
             OutputObj.outDate = ConfigOptions.current_fcst_cycle + datetime.timedelta(

core/geoMod.py

@@ -1,6 +1,11 @@
 import math
 
-import ESMF
+# ESMF was renamed to esmpy in v8.4.0
+try:
+    import esmpy as ESMF
+except ModuleNotFoundError:
+    import ESMF
+
 import numpy as np
 from netCDF4 import Dataset