PR: Fix water_level_measurements file management (#108)

* Removed dead imports

* Moved function to another module

Moved the function to create an empty water_levels_measures file to the
module read_waterlvl.

Also, removed the code to create an empty graph_layout.lst because it is
not used anymore.

* New water_level_measures and support for csv and xls

Added support for csv and xls file for reading the
water_level_measurement file.

* Added code to determine the ext of the file

* Added a property to return projet dirname

* Init the water_level_measurement on loading

When loading a project, the water_level_measurement is initialize
automatically by default.

* Make function xls/xlsx format aware

* cleaned not needed imports

* Added a basic test

* Fix a bug with extension search

* Fix when adding water lvl meas to project

The dataset size were not expanded to receive the data, so in the end,
no data was added to the dataset.

* init requirements file

* Added tests to test management of wl meas.

* Added a simple test for HydroCalc init

* Extended test coverture

gwhat/HydroPrint2.py

@@ -23,7 +23,6 @@
 
 # ---- Standard library imports
 
-import csv
 import sys
 import os
 
@@ -47,10 +46,8 @@
 from gwhat.meteo.weather_viewer import WeatherAvgGraph
 from gwhat.colors2 import ColorsReader, ColorsSetupWin
 
-from gwhat.common import IconDB, StyleDB, QToolButtonNormal, QToolButtonSmall
+from gwhat.common import IconDB, QToolButtonNormal, QToolButtonSmall
 import gwhat.common.widgets as myqt
-import gwhat.common.database as db
-from gwhat.common.utils import save_content_to_csv
 from gwhat.projet.reader_waterlvl import load_waterlvl_measures
 
 
@@ -88,8 +85,6 @@ def __init__(self, datamanager, parent=None):
 
         self.__initUI__()
 
-    # =========================================================================
-
     def __initUI__(self):
 
         # ---------------------------------------------------------- Toolbar --
@@ -464,34 +459,6 @@ def __init_labelLangWidget__(self):  # ------------------------------------
     def workdir(self):
         return self.dmngr.workdir
 
-    # ========================================================= Utilities =====
-
-    def check_files(self):
-
-        # water lvl manual measurements :
-
-        fname = os.path.join(self.workdir, 'waterlvl_manual_measurements.csv')
-        if not os.path.exists(fname):
-            msg = ('No "waterlvl_manual_measurements.xls" file found. '
-                   'A new one has been created.')
-            print(msg)
-
-            fcontent = [['Well_ID', 'Time (days)', 'Obs. (mbgs)']]
-            save_content_to_csv(fname, fcontent)
-
-        # graph_layout.lst :
-
-        filename = os.path.join(self.workdir, 'graph_layout.lst')
-        if not os.path.exists(filename):
-            msg = ('No "graph_layout.lst" file found. ' +
-                   'A new one has been created.')
-            print(msg)
-
-            fcontent = db.FileHeaders().graph_layout
-            save_content_to_csv(filename, fcontent)
-
-    # =========================================================================
-
     def zoom_in(self):
         self.hydrograph_scrollarea.zoomIn()
 
@@ -536,7 +503,8 @@ def wldset_changed(self):
 
         # Load Manual Measures :
 
-        fname = os.path.join(self.workdir, 'waterlvl_manual_measurements.xls')
+        fname = os.path.join(self.workdir, "Water Levels",
+                             'waterlvl_manual_measurements')
         tmeas, wlmeas = load_waterlvl_measures(fname, wldset['Well'])
         wldset.set_wlmeas(tmeas, wlmeas)
 
@@ -1184,15 +1152,13 @@ def show(self):
     ft.setPointSize(11)
     app.setFont(ft)
 
-    pf = ('C:/Users/jsgosselin/OneDrive/Research/'
-          'PostDoc - MDDELCC/Outils/BRF MontEst/'
-          'BRF MontEst.what')
+    pf = ("C:/Users/jsgosselin/GWHAT/gwhat/tests/"
+          "@ new-prô'jèt!/@ new-prô'jèt!.gwt")
     pr = ProjetReader(pf)
     dm = DataManager()
+    dm.set_projet(pr)
 
     Hydroprint = HydroprintGUI(dm)
     Hydroprint.show()
 
-    dm.set_projet(pr)
-
     sys.exit(app.exec_())

gwhat/projet/manager_projet.py

@@ -39,6 +39,7 @@
 from gwhat.projet.reader_projet import ProjetReader
 from gwhat.common import IconDB, QToolButtonSmall
 from gwhat.projet.manager_data import DataManager
+from gwhat.projet.reader_waterlvl import init_waterlvl_measures
 import gwhat.common.widgets as myqt
 from gwhat import __version__
 
@@ -110,6 +111,8 @@ def load_project(self, filename):
             QMessageBox.warning(self, 'Warning', msg, btn)
             return False
         else:
+            wldir = os.path.join(projet.dirname, "Water Levels")
+            init_waterlvl_measures(wldir)
             self.project_display.setText(projet.name)
             self.project_display.adjustSize()
             self.currentProjetChanged.emit(projet)

gwhat/projet/reader_projet.py

@@ -48,6 +48,10 @@ def db(self):  # project data base
     def filename(self):
         return self.db.filename
 
+    @property
+    def dirname(self):
+        return os.path.dirname(self.filename)
+
     # =========================================================================
 
     def load_projet(self, filename):
@@ -233,8 +237,8 @@ def add_wldset(self, name, df):
             # ---- Manual measurements ----
 
             mmeas = grp.create_group('manual')
-            mmeas.create_dataset('Time', data=np.array([]), maxshape=None)
-            mmeas.create_dataset('WL', data=np.array([]), maxshape=None)
+            mmeas.create_dataset('Time', data=np.array([]), maxshape=(None,))
+            mmeas.create_dataset('WL', data=np.array([]), maxshape=(None,))
 
             self.db.flush()
 
@@ -330,19 +334,28 @@ def __getitem__(self, key):
     def name(self):
         return self.dset.name
 
-    # =========================================================================
-
-    # manual measure get/set functions
+    # ---- Manual measurents
 
     def set_wlmeas(self, time, wl):
-        self.dset['manual/Time'][:] = time
-        self.dset['manual/WL'][:] = wl
+        """Overwrite the water level measurements for this dataset."""
+        try:
+            self.dset['manual/Time'].resize(np.shape(time))
+            self.dset['manual/Time'][:] = time
+            self.dset['manual/WL'].resize(np.shape(wl))
+            self.dset['manual/WL'][:] = wl
+        except TypeError:
+            del self.dset['manual']
+            mmeas = self.dset.create_group('manual')
+            mmeas.create_dataset('Time', data=time, maxshape=(None,))
+            mmeas.create_dataset('WL', data=wl, maxshape=(None,))
+        self.dset.file.flush()
 
     def get_wlmeas(self):
+        """Get the water level measurements for this dataset."""
         grp = self.dset.require_group('manual')
         return grp['Time'].value, grp['WL'].value
 
-    # =========================================================================
+    # ---- Master recession curve
 
     def set_mrc(self, A, B, peak_indx, time, recess):
         self.dset['mrc/params'][:] = (A, B)
@@ -375,7 +388,7 @@ def mrc_exists(self):
                                dtype='float64', maxshape=(None,))
         return bool(self.dset['mrc'].attrs['exists'])
 
-    # ================================================================ BRF ====
+    # ---- Barometric response function
 
     def saved_brf(self):
         grp = self.dset.require_group('brf')

gwhat/projet/reader_waterlvl.py

@@ -24,6 +24,11 @@
 import xlrd
 import csv
 
+# ---- Imports: local
+from gwhat.common.utils import save_content_to_csv
+
+FILE_EXTS = ['.csv', '.xls', '.xlsx']
+
 
 def load_excel_datafile(fname):
     print('Loading waterlvl time-series from Excel file...')
@@ -140,37 +145,78 @@ def make_waterlvl_continuous(t, wl):
 
     return t, wl
 
-# =============================================================================
-
-
-def load_waterlvl_measures(fname, well):
-
-    print('Loading waterlvl manual measures for well %s' % well)
-
-    WLmes, TIMEmes = [], []
-    if not os.path.exists(fname):
-        return WLmes, TIMEmes
-
-    with xlrd.open_workbook(fname) as wb:
-        sheet = wb.sheet_by_index(0)
-
-        NAME = sheet.col_values(0, start_rowx=1, end_rowx=None)
-        TIME = sheet.col_values(1, start_rowx=1, end_rowx=None)
-        OBS = sheet.col_values(2, start_rowx=1, end_rowx=None)
-
-        # Convert to Numpy :
-
-        NAME = np.array(NAME).astype('str')
-        TIME = np.array(TIME).astype('float')
-        OBS = np.array(OBS).astype('float')
-
-        if len(NAME) > 0:
-            rowx = np.where(NAME == well)[0]
-            if len(rowx) > 0:
-                WLmes = OBS[rowx]
-                TIMEmes = TIME[rowx]
 
-    return np.array(TIMEmes), np.array(WLmes)
+# ---- Water level manual measurements
+
+def init_waterlvl_measures(dirname):
+    """
+    Create an empty waterlvl_manual_measurements.csv file with headers
+    if it does not already exist.
+    """
+    for ext in FILE_EXTS:
+        fname = os.path.join(dirname, "waterlvl_manual_measurements"+ext)
+        if os.path.exists(fname):
+            return
+    else:
+        fname = os.path.join(dirname, 'waterlvl_manual_measurements.csv')
+        fcontent = [['Well_ID', 'Time (days)', 'Obs. (mbgs)']]
+
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        save_content_to_csv(fname, fcontent)
+
+
+def load_waterlvl_measures(filename, well):
+    """
+    Load and read the water level manual measurements from the specified
+    resource file for the specified well.
+    """
+    print('Loading manual water level measures for well %s...' % well, end=" ")
+    time_mes, wl_mes = np.array([]), np.array([])
+    # Determine the extension of the file.
+    root, ext = os.path.splitext(filename)
+    exts = [ext] if ext in FILE_EXTS else FILE_EXTS
+    for ext in exts:
+        filename = root+ext
+        if os.path.exists(root+ext):
+            break
+    else:
+        # The file does not exists, so we generate an empty file with
+        # a header.
+        print("none")
+        init_waterlvl_measures(os.path.dirname(root))
+        return time_mes, wl_mes
+
+    # Open and read the file.
+    if ext == '.csv':
+        with open(filename, 'r') as f:
+            reader = np.array(list(csv.reader(f, delimiter=',')))
+            data = np.array(reader[1:])
+
+            well_name = np.array(data[:, 0]).astype('str')
+            time = np.array(data[:, 1]).astype('float')
+            wl = np.array(data[:, 2]).astype('float')
+
+    elif ext in ['.xlsx', '.xls']:
+        with xlrd.open_workbook(filename) as wb:
+            sheet = wb.sheet_by_index(0)
+
+            well_name = sheet.col_values(0, start_rowx=1, end_rowx=None)
+            time = sheet.col_values(1, start_rowx=1, end_rowx=None)
+            wl = sheet.col_values(2, start_rowx=1, end_rowx=None)
+
+            well_name = np.array(well_name).astype('str')
+            time = np.array(time).astype('float')
+            wl = np.array(wl).astype('float')
+
+    if len(well_name) > 0:
+        rowx = np.where(well_name == well)[0]
+        if len(rowx) > 0:
+            wl_mes = wl[rowx]
+            time_mes = time[rowx]
+    print("done")
+
+    return time_mes, wl_mes
 
 
 # =========================================================================

gwhat/tests/pytest_test_order.txt

@@ -5,5 +5,8 @@
 5 - test_gapfill_weather_gui.py
 6 - test_gapfill_weather_algorithm.py
 7 - test_manager_data.py
+8 - test_hydroprint.py
+9 - test_hydrocalc.py
 
-test_tabwidget.py
+test_tabwidget.py
+test_read_waterlvl.py

gwhat/tests/test_hydrocalc.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014-2017 GWHAT Project Contributors
+# https://github.com/jnsebgosselin/gwhat
+#
+# This file is part of GWHAT (GroundWater Hydrograph Analysis Toolbox).
+# Licensed under the terms of the GNU General Public License.
+
+# Standard library imports
+import sys
+import os
+
+# Third party imports
+import pytest
+from PyQt5.QtCore import Qt
+
+# Local imports
+sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+from gwhat.HydroCalc2 import WLCalc
+from gwhat.projet.manager_data import DataManager
+from gwhat.projet.reader_projet import ProjetReader
+
+
+# Qt Test Fixtures
+# --------------------------------
+
+
+working_dir = os.path.join(os.getcwd(), "@ new-prô'jèt!")
+output_dir = os.path.join(working_dir, "Water Levels")
+
+
+@pytest.fixture
+def hydrocalc_bot(qtbot):
+    pf = os.path.join(working_dir, "@ new-prô'jèt!.gwt")
+    pr = ProjetReader(pf)
+
+    dm = DataManager()
+    dm.set_projet(pr)
+
+    hydrocalc = WLCalc(dm)
+    qtbot.addWidget(hydrocalc)
+
+    return hydrocalc, qtbot
+
+
+# Test WLCalc
+# -------------------------------
+
+
+@pytest.mark.run(order=9)
+def test_hydrocalc_init(hydrocalc_bot, mocker):
+    hydrocalc, qtbot = hydrocalc_bot
+    hydrocalc.show()
+    assert hydrocalc
+
+
+if __name__ == "__main__":
+    pytest.main(['-x', os.path.basename(__file__), '-v', '-rw'])
+    # pytest.main()