PR: Store water levels in a Pandas DataFrame and index time with datetime instead of Excel numerical date

gwhat/HydroCalc2.py

@@ -519,7 +519,7 @@ def water_lvl(self):
 
     @property
     def time(self):
-        return np.array([]) if self.wldset is None else self.wldset.datetimes
+        return np.array([]) if self.wldset is None else self.wldset.xldates
 
     @property
     def wldset(self):
@@ -1228,8 +1228,8 @@ def _draw_obs_wl(self, draw=True):
         self.clear_selected_wl(draw=False)
         if self.wldset is not None:
             self._obs_wl_plt.set_data(
-                self.wldset.datetimes + (self.dt4xls2mpl * self.dformat),
-                self.wldset.waterlevels)
+                self.time + (self.dt4xls2mpl * self.dformat),
+                self.water_lvl)
         self._obs_wl_plt.set_visible(self.wldset is not None)
         if draw:
             self.draw()

gwhat/HydroPrint2.py

@@ -480,7 +480,7 @@ def best_fit_waterlvl(self):
     def best_fit_time(self):
         wldset = self.dmngr.get_current_wldset()
         if wldset is not None:
-            date0, date1 = self.hydrograph.best_fit_time(wldset['Time'])
+            date0, date1 = self.hydrograph.best_fit_time(wldset.xldates)
             self.date_start_widget.setDate(QDate(date0[0], date0[1], date0[2]))
             self.date_end_widget.setDate(QDate(date1[0], date1[1], date1[2]))
 

gwhat/brf_mod/kgs_gui.py

@@ -306,15 +306,13 @@ def set_wldset(self, wldset):
         self.btn_seldata.setAutoRaise(True)
         self.setEnabled(wldset is not None)
         if wldset is not None:
-            self.set_daterange((self.wldset['Time'][0],
-                                self.wldset['Time'][-1]))
+            xldates = self.wldset.xldates
+            self.set_daterange((xldates[0], xldates[-1]))
 
             # Set the period over which the BRF would be evaluated.
             saved_brfperiod = wldset.get_brfperiod()
-            self.set_brfperiod(
-                (saved_brfperiod[0] or np.floor(self.wldset['Time'][0]),
-                 saved_brfperiod[1] or np.floor(self.wldset['Time'][-1])
-                 ))
+            self.set_brfperiod((saved_brfperiod[0] or np.floor(xldates[0]),
+                                saved_brfperiod[1] or np.floor(xldates[-1])))
 
     def set_daterange(self, daterange):
         """
@@ -335,11 +333,11 @@ def calc_brf(self):
 
         brfperiod = self.get_brfperiod()
         t1 = min(brfperiod)
-        i1 = np.where(self.wldset['Time'] >= t1)[0][0]
+        i1 = np.where(self.wldset.xldates >= t1)[0][0]
         t2 = max(brfperiod)
-        i2 = np.where(self.wldset['Time'] <= t2)[0][-1]
+        i2 = np.where(self.wldset.xldates <= t2)[0][-1]
 
-        time = np.copy(self.wldset['Time'][i1:i2+1])
+        time = np.copy(self.wldset.xldates[i1:i2+1])
         wl = np.copy(self.wldset['WL'][i1:i2+1])
         bp = np.copy(self.wldset['BP'][i1:i2+1])
         if len(bp) == 0:

gwhat/gwrecharge/gwrecharge_calc2.py

@@ -98,8 +98,8 @@ def load_data(self, wxdset, wldset):
 
         self.wldset = wldset
         self.A, self.B = wldset['mrc/params']
-        self.twlvl, self.wlobs = self.make_data_daily(wldset['Time'],
-                                                      wldset['WL'])
+        self.twlvl, self.wlobs = self.make_data_daily(
+            wldset.xldate, wldset['WL'])
 
         if not self.A and not self.B:
             error = ("Groundwater recharge cannot be computed because a"

gwhat/hydrograph4.py

@@ -855,7 +855,7 @@ def draw_waterlvl(self):
 
         # ---- Logger Measures
 
-        time = self.wldset['Time']
+        time = self.wldset.xldates
         if self.WLdatum == 1:  # masl
             water_lvl = self.wldset['Elevation']-self.wldset['WL']
         else:  # mbgs -> yaxis is inverted
@@ -1333,7 +1333,7 @@ def LatLong2Dist(LAT1, LON1, LAT2, LON2):
         # 0: daily | 1: weekly | 2: monthly | 3: yearly
         hydrograph.RAINscale = 100
 
-        hydrograph.best_fit_time(wldset['Time'])
+        hydrograph.best_fit_time(wldset.xldate)
         hydrograph.best_fit_waterlvl()
         hydrograph.generate_hydrograph()
 #

gwhat/projet/manager_data.py

@@ -24,10 +24,10 @@
 from gwhat.utils import icons
 import gwhat.common.widgets as myqt
 from gwhat.hydrograph4 import LatLong2Dist
-import gwhat.projet.reader_waterlvl as wlrd
+from gwhat.projet.reader_waterlvl import WLDataFrame
 from gwhat.projet.reader_projet import (INVALID_CHARS, is_dsetname_valid,
                                         make_dsetname_valid)
-import gwhat.meteo.weather_reader as wxrd
+from gwhat.meteo.weather_reader import WXDataFrame
 from gwhat.widgets.buttons import ToolBarWidget
 from gwhat.widgets.spinboxes import StrSpinBox
 
@@ -698,9 +698,9 @@ def load_dataset(self, filename):
 
         try:
             if self._datatype == 'water level':
-                self._dataset = wlrd.read_water_level_datafile(filename)
+                self._dataset = WLDataFrame(filename)
             elif self._datatype == 'daily weather':
-                self._dataset = wxrd.WXDataFrame(filename)
+                self._dataset = WXDataFrame(filename)
         except Exception:
             self._dataset = None
 

gwhat/projet/reader_projet.py

@@ -10,7 +10,6 @@
 
 # ---- Standard library imports
 import os
-import csv
 import os.path as osp
 from shutil import copyfile
 
@@ -20,7 +19,7 @@
 
 # ---- Local library imports
 from gwhat.meteo.weather_reader import WXDataFrameBase
-from gwhat.projet.reader_waterlvl import WLDataFrameBase
+from gwhat.projet.reader_waterlvl import WLDataFrameBase, WLDataset
 from gwhat.gwrecharge.glue import GLUEDataFrameBase
 from gwhat.common.utils import save_content_to_file
 from gwhat.utils.math import nan_as_text_tolist, calcul_rmse
@@ -124,7 +123,6 @@ def backup_project_file(self):
             print('done')
             return True
 
-
     # ---- Project Properties
     @property
     def name(self):
@@ -183,7 +181,6 @@ def lon(self, x):
         self.db.attrs['longitude'] = x
 
     # ---- Water Levels Dataset Handlers
-
     @property
     def wldsets(self):
         """
@@ -224,10 +221,16 @@ def add_wldset(self, name, df):
             grp = self.db['wldsets'].create_group(name)
 
             # Water level data
-            grp.create_dataset('Time', data=df['Time'])
-            grp.create_dataset('WL', data=df['WL'])
-            grp.create_dataset('BP', data=df['BP'])
-            grp.create_dataset('ET', data=df['ET'])
+            grp.create_dataset(
+                'Time',
+                data=np.array(df['Time'], dtype=h5py.special_dtype(vlen=str)))
+            # See http://docs.h5py.org/en/latest/strings.html as to why this
+            # is necessary to do this in order to save a list of strings in
+            # a dataset with h5py.
+
+            grp.create_dataset('WL', data=np.copy(df['WL']))
+            grp.create_dataset('BP', data=np.copy(df['BP']))
+            grp.create_dataset('ET', data=np.copy(df['ET']))
 
             # Piezometric well info
             grp.attrs['filename'] = df['filename']
@@ -280,7 +283,6 @@ def del_wldset(self, name):
         self.db.flush()
 
     # ---- Weather Dataset Handlers
-
     @property
     def wxdsets(self):
         """
@@ -381,8 +383,16 @@ def __init__(self, hdf5group, *args, **kwargs):
     def __load_dataset__(self, hdf5group):
         self.dset = hdf5group
         self._undo_stack = []
-        self._waterlevels = self.dset['WL'][...]
-        self._datetimes = self.dset['Time'][...]
+
+        columns = []
+        data = []
+        for colname in ['Time', 'WL', 'BP', 'ET']:
+            if len(self.dset[colname][...]):
+                data.append(self.dset[colname][...])
+                columns.append(colname)
+        data = np.vstack(tuple(data)).transpose()
+        columns = tuple(columns)
+        self._dataf = WLDataset(data, columns)
 
         # Make older datasets compatible with newer format.
         if 'Well ID' not in list(self.dset.attrs.keys()):