diff --git a/.travis.yml b/.travis.yml
index cc6fc9c87..3c31cd283 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,6 +19,7 @@ install:
- pip install pytest-ordering
- pip install coveralls
- pip install matplotlib
+ - pip install scipy
- pip install beautifulsoup4
script:
diff --git a/appveyor.yml b/appveyor.yml
index 9b8ae4c85..c7350e1f6 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -39,6 +39,7 @@ install:
- python -m pip install pytest-ordering
- python -m pip install coveralls
- python -m pip install matplotlib
+ - python -m pip install scipy
- python -m pip install beautifulsoup4
build: false
diff --git a/gwhat/common/utils.py b/gwhat/common/utils.py
index 87d021a28..f67655475 100644
--- a/gwhat/common/utils.py
+++ b/gwhat/common/utils.py
@@ -10,6 +10,7 @@
# ---- Imports: standard libraries
import csv
+import os
# ---- Imports: third party
@@ -39,3 +40,13 @@ def save_content_to_csv(fname, fcontent, mode='w', delimiter=','):
with open(fname, mode) as csvfile:
writer = csv.writer(csvfile, delimiter=delimiter, lineterminator='\n')
writer.writerows(fcontent)
+
+
+def delete_file(filename):
+ """Try to delete a file on the disk and return the error if any."""
+ try:
+ os.remove(filename)
+ return None
+ except OSError as e:
+ print("Error: %s - %s." % (e.filename, e.strerror))
+ return e.strerror
diff --git a/gwhat/hydrograph4.py b/gwhat/hydrograph4.py
index 1553bba0d..663e61ae3 100644
--- a/gwhat/hydrograph4.py
+++ b/gwhat/hydrograph4.py
@@ -45,8 +45,7 @@
import gwhat.common.database as db
from gwhat.colors2 import ColorsReader
-mpl.use('Qt4Agg')
-mpl.rcParams['backend.qt4'] = 'PySide'
+mpl.use('Qt5Agg')
mpl.rc('font', **{'family': 'sans-serif', 'sans-serif': ['Arial']})
diff --git a/gwhat/meteo/gapfill_weather_algorithm2.py b/gwhat/meteo/gapfill_weather_algorithm2.py
index 7b5520efe..38af2dbaf 100644
--- a/gwhat/meteo/gapfill_weather_algorithm2.py
+++ b/gwhat/meteo/gapfill_weather_algorithm2.py
@@ -145,7 +145,7 @@ def load_data(self):
if not os.path.exists(binfile):
return self.reload_data()
- # ---- Scan input folder for changes ----------------------------------
+ # ---- Scan input folder for changes
# If one of the csv data file contained within the input data directory
# has changed since last time the binary file was created, the
@@ -156,12 +156,18 @@ def load_data(self):
fnames = A['fnames']
bmtime = os.path.getmtime(binfile)
+ count = 0
for f in os.listdir(self.inputDir):
if f.endswith('.csv'):
+ count += 1
fmtime = os.path.getmtime(os.path.join(self.inputDir, f))
if f not in fnames or fmtime > bmtime:
return self.reload_data()
+ # Force a reload of the data if some input files were deleted.
+ if len(fnames) != count:
+ return self.reload_data()
+
# ---- Load data from binary ------------------------------------------
print('\nLoading data from binary file :\n')
@@ -183,11 +189,10 @@ def reload_data(self):
n = len(paths)
print('\n%d valid weather data files found in Input folder.' % n)
- print('Loading data from csv files :\n')
-
+ print('Loading data from csv files...')
self.WEATHER.load_and_format_data(paths)
self.WEATHER.save_to_binary(self.inputDir)
-
+ print('Data loaded sucessfully.')
self.WEATHER.generate_summary(self.outputDir)
self.TARGET.index = -1
@@ -1434,7 +1439,6 @@ def load_and_format_data(self, paths):
# Check if data are continuous over time. If not, the serie will be
# made continuous and the gaps will be filled with nan values.
- print(reader[0][1])
time_start = xldate_from_date_tuple((STADAT[0, 0].astype('int'),
STADAT[0, 1].astype('int'),
@@ -1446,9 +1450,6 @@ def load_and_format_data(self, paths):
STADAT[-1, 2].astype('int')),
0)
- print(time_start, time_end, len(STADAT[:, 0]))
- print(time_end - time_start + 1)
-
if (time_end - time_start + 1) != len(STADAT[:, 0]):
print('\n%s is not continuous, correcting...' % reader[0][1])
STADAT = self.make_timeserie_continuous(STADAT)
diff --git a/gwhat/meteo/gapfill_weather_gui.py b/gwhat/meteo/gapfill_weather_gui.py
index 2b837cc4e..c1df64f76 100644
--- a/gwhat/meteo/gapfill_weather_gui.py
+++ b/gwhat/meteo/gapfill_weather_gui.py
@@ -29,6 +29,7 @@
# ---- Third party imports
+from PyQt5.QtCore import pyqtSlot as QSlot
from PyQt5.QtCore import pyqtSignal as QSignal
from PyQt5.QtCore import Qt, QThread, QDate, QRect
from PyQt5.QtGui import QBrush, QColor, QFont, QPainter, QCursor, QTextDocument
@@ -54,6 +55,7 @@
from gwhat.meteo.merge_weather_data import WXDataMergerWidget
from gwhat.common import IconDB, StyleDB, QToolButtonSmall
import gwhat.common.widgets as myqt
+from gwhat.common.utils import delete_file
class GapFillWeatherGUI(QWidget):
@@ -82,7 +84,7 @@ def __init__(self, parent=None):
def __initUI__(self):
self.setWindowIcon(IconDB().master)
- # ---- TOOLBAR ----
+ # ---- Toolbar at the bottom
self.btn_fill = QPushButton('Fill Station')
self.btn_fill.setIcon(IconDB().fill_data)
@@ -113,13 +115,12 @@ def __initUI__(self):
widget_toolbar.setLayout(grid_toolbar)
- # ----------------------------------------------------- LEFT PANEL ----
+ # ---- Target Station groupbox
- # ---- Target Station ----
-
- target_station_label = QLabel(
- 'Fill data for weather station :')
self.target_station = QComboBox()
+ self.target_station.currentIndexChanged.connect(
+ self.target_station_changed)
+
self.target_station_info = QTextEdit()
self.target_station_info.setReadOnly(True)
self.target_station_info.setMaximumHeight(110)
@@ -127,26 +128,38 @@ def __initUI__(self):
self.btn_refresh_staList = QToolButtonSmall(IconDB().refresh)
self.btn_refresh_staList.setToolTip(
'Force the reloading of the weather data files')
- self.btn_refresh_staList.clicked.connect(self.load_data_dir_content)
+ self.btn_refresh_staList.clicked.connect(self.btn_refresh_isclicked)
btn_merge_data = QToolButtonSmall(IconDB().merge_data)
btn_merge_data.setToolTip(
'Tool for merging two ore more datasets together.')
btn_merge_data.clicked.connect(self.wxdata_merger.show)
+ self.btn_delete_data = QToolButtonSmall(IconDB().clear)
+ self.btn_delete_data.setEnabled(False)
+ self.btn_delete_data.setToolTip(
+ 'Remove the currently selected dataset and delete the input '
+ 'datafile. However, raw datafiles will be kept.')
+ self.btn_delete_data.clicked.connect(self.delete_current_dataset)
+
+ widgets = [self.target_station, self.btn_refresh_staList,
+ btn_merge_data, self.btn_delete_data]
+
# Generate the layout for the target station group widget.
self.tarSta_widg = QWidget()
tarSta_grid = QGridLayout(self.tarSta_widg)
row = 0
- tarSta_grid.addWidget(target_station_label, row, 0, 1, 3)
- row = 1
- tarSta_grid.addWidget(self.target_station, row, 0)
- tarSta_grid.addWidget(self.btn_refresh_staList, row, 1)
- tarSta_grid.addWidget(btn_merge_data, row, 2)
- row = 2
- tarSta_grid.addWidget(self.target_station_info, row, 0, 1, 3)
+ tarSta_grid.addWidget(QLabel('Fill data for weather station :'),
+ row, 0, 1, len(widgets))
+ row += 1
+ tarSta_grid.addWidget(self.target_station, 1, 0)
+ for col, widget in enumerate(widgets):
+ tarSta_grid.addWidget(widget, row, col)
+ row += 1
+ tarSta_grid.addWidget(self.target_station_info,
+ row, 0, 1, len(widgets))
tarSta_grid.setSpacing(5)
tarSta_grid.setColumnStretch(0, 500)
@@ -314,7 +327,7 @@ def advanced_settings(self):
self.stack_widget.addItem(MLRM_widg, 'Regression Model :')
self.stack_widget.addItem(advanced_widg, 'Advanced Settings :')
- # SUBGRIDS ASSEMBLY :
+ # ---- LEFT PANEL
grid_leftPanel = QGridLayout()
self.LEFT_widget = QFrame()
@@ -340,7 +353,7 @@ def advanced_settings(self):
self.LEFT_widget.setLayout(grid_leftPanel)
- # ---- Right Panel ----
+ # ---- Right Panel
self.FillTextBox = QTextEdit()
self.FillTextBox.setReadOnly(True)
@@ -382,7 +395,7 @@ def advanced_settings(self):
# RIGHT_widget.addTab(self.gafill_display_table,
# 'New Table (Work-in-Progress)')
- # ---- Main grid ----
+ # ---- Main grid
grid_MAIN = QGridLayout()
@@ -397,17 +410,16 @@ def advanced_settings(self):
self.setLayout(grid_MAIN)
- # ---- Progress Bar ----
+ # ---- Progress Bar
self.pbar = QProgressBar()
self.pbar.setValue(0)
self.pbar.hide()
- # ---- Events ----
+ # ---- Events
# CORRELATION :
- self.target_station.currentIndexChanged.connect(self.correlation_UI)
self.distlimit.valueChanged.connect(self.correlation_table_display)
self.altlimit.valueChanged.connect(self.correlation_table_display)
self.date_start_widget.dateChanged.connect(
@@ -439,14 +451,32 @@ def set_workdir(self, dirname):
self.wxdata_merger.set_workdir(os.path.join(dirname, 'Meteo', 'Input'))
- # =========================================================================
+ def delete_current_dataset(self):
+ """
+ Delete the current dataset source file and force a reload of the input
+ daily weather datafiles.
+ """
+ current_index = self.target_station.currentIndex()
+ if current_index != -1:
+ basename = self.gapfill_worker.WEATHER.fnames[current_index]
+ dirname = self.gapfill_worker.inputDir
+ filename = os.path.join(dirname, basename)
+ delete_file(filename)
+ self.load_data_dir_content(reload=True)
+
+ def btn_refresh_isclicked(self):
+ """
+ Handles when the button to refresh the list of input daily weather
+ datafiles is clicked
+ """
+ self.load_data_dir_content(reload=True)
- def load_data_dir_content(self):
- '''
+ def load_data_dir_content(self, reload=False):
+ """
Initiate the loading of Weater Data Files contained in the
- */Meteo/Input* folder and display the resulting station list in the
- *Target station* combo box widget.
- '''
+ */Meteo/Input folder and display the resulting station list in the
+ target station combobox.
+ """
# Reset UI :
@@ -460,10 +490,11 @@ def load_data_dir_content(self):
self.CORRFLAG = 'off'
# Correlation calculation won't be triggered when this is off
- if self.sender() == self.btn_refresh_staList:
+ if reload:
stanames = self.gapfill_worker.reload_data()
else:
stanames = self.gapfill_worker.load_data()
+
self.target_station.addItems(stanames)
self.target_station.setCurrentIndex(-1)
self.sta_display_summary.setHtml(self.gapfill_worker.read_summary())
@@ -535,16 +566,20 @@ def correlation_table_display(self): # ===================================
self.FillTextBox.setText(table)
self.target_station_info.setText(target_info)
- def correlation_UI(self): # ==============================================
+ @QSlot(int)
+ def target_station_changed(self, index):
+ """Handles when the target station is changed on the GUI side."""
+ self.btn_delete_data.setEnabled(index != -1)
+ if index != -1:
+ self.correlation_UI()
+ def correlation_UI(self):
"""
Calculate automatically the correlation coefficients when a target
station is selected by the user in the drop-down menu or if a new
station is selected programmatically.
"""
-
if self.CORRFLAG == 'on' and self.target_station.currentIndex() != -1:
-
index = self.target_station.currentIndex()
self.gapfill_worker.set_target_station(index)
@@ -1529,8 +1564,7 @@ def sizeHint(self):
app.setFont(QFont('Ubuntu', 11))
w = GapFillWeatherGUI()
- w.set_workdir("C:\\Users\\jsgosselin\\OneDrive\\WHAT"
- "\\WHAT\\tests\\@ new-prô'jèt!")
+ w.set_workdir("C:\\Users\\jsgosselin\\OneDrive\\GWHAT\\Projects\\Example")
w.load_data_dir_content()
lat = w.gapfill_worker.WEATHER.LAT
diff --git a/gwhat/tests/test_gapfill_weather_data.py b/gwhat/tests/test_gapfill_weather_data.py
new file mode 100644
index 000000000..11798d608
--- /dev/null
+++ b/gwhat/tests/test_gapfill_weather_data.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Aug 4 01:50:50 2017
+@author: jsgosselin
+"""
+
+# Standard library imports
+import sys
+import os
+
+# Third party imports
+import numpy as np
+from numpy import nan
+import pytest
+from PyQt5.QtCore import Qt
+
+# Local imports
+sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+from gwhat.meteo.gapfill_weather_gui import GapFillWeatherGUI
+
+
+# Qt Test Fixtures
+# --------------------------------
+
+
+working_dir = os.path.join(os.getcwd(), "@ new-prô'jèt!")
+
+
+@pytest.fixture
+def gapfill_weather_bot(qtbot):
+ gapfiller = GapFillWeatherGUI()
+ gapfiller.set_workdir(working_dir)
+ qtbot.addWidget(gapfiller)
+
+ return gapfiller, qtbot
+
+
+# Test RawDataDownloader
+# -------------------------------
+
+expected_results = ["IBERVILLE", "IBERVILLE (1)",
+ "L'ACADIE", "L'ACADIE (1)",
+ "MARIEVILLE", "MARIEVILLE (1)",
+ "Station 1", "Station 1 (1)"]
+
+
+@pytest.mark.run(order=5)
+def test_refresh_data(gapfill_weather_bot, mocker):
+ gapfiller, qtbot = gapfill_weather_bot
+ gapfiller.show()
+
+ # Load the input weather datafiles and assert that the list is loaded and
+ # displayed as expected.
+ qtbot.mouseClick(gapfiller.btn_refresh_staList, Qt.LeftButton)
+
+ results = []
+ for i in range(gapfiller.target_station.count()):
+ results.append(gapfiller.target_station.itemText(i))
+
+ assert expected_results == results
+
+
+@pytest.mark.run(order=5)
+def test_delete_data(gapfill_weather_bot, mocker):
+ gapfiller, qtbot = gapfill_weather_bot
+ gapfiller.show()
+
+ # Load the input weather datafiles and select the last one in the list.
+ qtbot.mouseClick(gapfiller.btn_refresh_staList, Qt.LeftButton)
+ last_index = gapfiller.target_station.count()-1
+ gapfiller.target_station.setCurrentIndex(last_index)
+ assert gapfiller.target_station.currentText() == expected_results[-1]
+
+ # Delete the currently selected dataset.
+ qtbot.mouseClick(gapfiller.btn_delete_data, Qt.LeftButton)
+
+ results = []
+ for i in range(gapfiller.target_station.count()):
+ results.append(gapfiller.target_station.itemText(i))
+
+ assert expected_results[:-1] == results
+
+
+if __name__ == "__main__":
+ pytest.main([os.path.basename(__file__)])
+ # pytest.main()
diff --git a/runtests.py b/runtests.py
index 910631f4a..2eb69824c 100644
--- a/runtests.py
+++ b/runtests.py
@@ -9,6 +9,8 @@
"""
import pytest
+import matplotlib as mpl
+mpl.use('Qt5Agg')
def main():