diff --git a/.travis.yml b/.travis.yml index cc6fc9c87..3c31cd283 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ install: - pip install pytest-ordering - pip install coveralls - pip install matplotlib + - pip install scipy - pip install beautifulsoup4 script: diff --git a/appveyor.yml b/appveyor.yml index 9b8ae4c85..c7350e1f6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -39,6 +39,7 @@ install: - python -m pip install pytest-ordering - python -m pip install coveralls - python -m pip install matplotlib + - python -m pip install scipy - python -m pip install beautifulsoup4 build: false diff --git a/gwhat/common/utils.py b/gwhat/common/utils.py index 87d021a28..f67655475 100644 --- a/gwhat/common/utils.py +++ b/gwhat/common/utils.py @@ -10,6 +10,7 @@ # ---- Imports: standard libraries import csv +import os # ---- Imports: third party @@ -39,3 +40,13 @@ def save_content_to_csv(fname, fcontent, mode='w', delimiter=','): with open(fname, mode) as csvfile: writer = csv.writer(csvfile, delimiter=delimiter, lineterminator='\n') writer.writerows(fcontent) + + +def delete_file(filename): + """Try to delete a file on the disk and return the error if any.""" + try: + os.remove(filename) + return None + except OSError as e: + print("Error: %s - %s." % (e.filename, e.strerror)) + return e.strerror diff --git a/gwhat/hydrograph4.py b/gwhat/hydrograph4.py index 1553bba0d..663e61ae3 100644 --- a/gwhat/hydrograph4.py +++ b/gwhat/hydrograph4.py @@ -45,8 +45,7 @@ import gwhat.common.database as db from gwhat.colors2 import ColorsReader -mpl.use('Qt4Agg') -mpl.rcParams['backend.qt4'] = 'PySide' +mpl.use('Qt5Agg') mpl.rc('font', **{'family': 'sans-serif', 'sans-serif': ['Arial']}) diff --git a/gwhat/meteo/gapfill_weather_algorithm2.py b/gwhat/meteo/gapfill_weather_algorithm2.py index 7b5520efe..38af2dbaf 100644 --- a/gwhat/meteo/gapfill_weather_algorithm2.py +++ b/gwhat/meteo/gapfill_weather_algorithm2.py @@ -145,7 +145,7 @@ def load_data(self): if not os.path.exists(binfile): return self.reload_data() - # ---- Scan input folder for changes ---------------------------------- + # ---- Scan input folder for changes # If one of the csv data file contained within the input data directory # has changed since last time the binary file was created, the @@ -156,12 +156,18 @@ def load_data(self): fnames = A['fnames'] bmtime = os.path.getmtime(binfile) + count = 0 for f in os.listdir(self.inputDir): if f.endswith('.csv'): + count += 1 fmtime = os.path.getmtime(os.path.join(self.inputDir, f)) if f not in fnames or fmtime > bmtime: return self.reload_data() + # Force a reload of the data if some input files were deleted. + if len(fnames) != count: + return self.reload_data() + # ---- Load data from binary ------------------------------------------ print('\nLoading data from binary file :\n') @@ -183,11 +189,10 @@ def reload_data(self): n = len(paths) print('\n%d valid weather data files found in Input folder.' % n) - print('Loading data from csv files :\n') - + print('Loading data from csv files...') self.WEATHER.load_and_format_data(paths) self.WEATHER.save_to_binary(self.inputDir) - + print('Data loaded sucessfully.') self.WEATHER.generate_summary(self.outputDir) self.TARGET.index = -1 @@ -1434,7 +1439,6 @@ def load_and_format_data(self, paths): # Check if data are continuous over time. If not, the serie will be # made continuous and the gaps will be filled with nan values. - print(reader[0][1]) time_start = xldate_from_date_tuple((STADAT[0, 0].astype('int'), STADAT[0, 1].astype('int'), @@ -1446,9 +1450,6 @@ def load_and_format_data(self, paths): STADAT[-1, 2].astype('int')), 0) - print(time_start, time_end, len(STADAT[:, 0])) - print(time_end - time_start + 1) - if (time_end - time_start + 1) != len(STADAT[:, 0]): print('\n%s is not continuous, correcting...' % reader[0][1]) STADAT = self.make_timeserie_continuous(STADAT) diff --git a/gwhat/meteo/gapfill_weather_gui.py b/gwhat/meteo/gapfill_weather_gui.py index 2b837cc4e..c1df64f76 100644 --- a/gwhat/meteo/gapfill_weather_gui.py +++ b/gwhat/meteo/gapfill_weather_gui.py @@ -29,6 +29,7 @@ # ---- Third party imports +from PyQt5.QtCore import pyqtSlot as QSlot from PyQt5.QtCore import pyqtSignal as QSignal from PyQt5.QtCore import Qt, QThread, QDate, QRect from PyQt5.QtGui import QBrush, QColor, QFont, QPainter, QCursor, QTextDocument @@ -54,6 +55,7 @@ from gwhat.meteo.merge_weather_data import WXDataMergerWidget from gwhat.common import IconDB, StyleDB, QToolButtonSmall import gwhat.common.widgets as myqt +from gwhat.common.utils import delete_file class GapFillWeatherGUI(QWidget): @@ -82,7 +84,7 @@ def __init__(self, parent=None): def __initUI__(self): self.setWindowIcon(IconDB().master) - # ---- TOOLBAR ---- + # ---- Toolbar at the bottom self.btn_fill = QPushButton('Fill Station') self.btn_fill.setIcon(IconDB().fill_data) @@ -113,13 +115,12 @@ def __initUI__(self): widget_toolbar.setLayout(grid_toolbar) - # ----------------------------------------------------- LEFT PANEL ---- + # ---- Target Station groupbox - # ---- Target Station ---- - - target_station_label = QLabel( - 'Fill data for weather station :') self.target_station = QComboBox() + self.target_station.currentIndexChanged.connect( + self.target_station_changed) + self.target_station_info = QTextEdit() self.target_station_info.setReadOnly(True) self.target_station_info.setMaximumHeight(110) @@ -127,26 +128,38 @@ def __initUI__(self): self.btn_refresh_staList = QToolButtonSmall(IconDB().refresh) self.btn_refresh_staList.setToolTip( 'Force the reloading of the weather data files') - self.btn_refresh_staList.clicked.connect(self.load_data_dir_content) + self.btn_refresh_staList.clicked.connect(self.btn_refresh_isclicked) btn_merge_data = QToolButtonSmall(IconDB().merge_data) btn_merge_data.setToolTip( 'Tool for merging two ore more datasets together.') btn_merge_data.clicked.connect(self.wxdata_merger.show) + self.btn_delete_data = QToolButtonSmall(IconDB().clear) + self.btn_delete_data.setEnabled(False) + self.btn_delete_data.setToolTip( + 'Remove the currently selected dataset and delete the input ' + 'datafile. However, raw datafiles will be kept.') + self.btn_delete_data.clicked.connect(self.delete_current_dataset) + + widgets = [self.target_station, self.btn_refresh_staList, + btn_merge_data, self.btn_delete_data] + # Generate the layout for the target station group widget. self.tarSta_widg = QWidget() tarSta_grid = QGridLayout(self.tarSta_widg) row = 0 - tarSta_grid.addWidget(target_station_label, row, 0, 1, 3) - row = 1 - tarSta_grid.addWidget(self.target_station, row, 0) - tarSta_grid.addWidget(self.btn_refresh_staList, row, 1) - tarSta_grid.addWidget(btn_merge_data, row, 2) - row = 2 - tarSta_grid.addWidget(self.target_station_info, row, 0, 1, 3) + tarSta_grid.addWidget(QLabel('Fill data for weather station :'), + row, 0, 1, len(widgets)) + row += 1 + tarSta_grid.addWidget(self.target_station, 1, 0) + for col, widget in enumerate(widgets): + tarSta_grid.addWidget(widget, row, col) + row += 1 + tarSta_grid.addWidget(self.target_station_info, + row, 0, 1, len(widgets)) tarSta_grid.setSpacing(5) tarSta_grid.setColumnStretch(0, 500) @@ -314,7 +327,7 @@ def advanced_settings(self): self.stack_widget.addItem(MLRM_widg, 'Regression Model :') self.stack_widget.addItem(advanced_widg, 'Advanced Settings :') - # SUBGRIDS ASSEMBLY : + # ---- LEFT PANEL grid_leftPanel = QGridLayout() self.LEFT_widget = QFrame() @@ -340,7 +353,7 @@ def advanced_settings(self): self.LEFT_widget.setLayout(grid_leftPanel) - # ---- Right Panel ---- + # ---- Right Panel self.FillTextBox = QTextEdit() self.FillTextBox.setReadOnly(True) @@ -382,7 +395,7 @@ def advanced_settings(self): # RIGHT_widget.addTab(self.gafill_display_table, # 'New Table (Work-in-Progress)') - # ---- Main grid ---- + # ---- Main grid grid_MAIN = QGridLayout() @@ -397,17 +410,16 @@ def advanced_settings(self): self.setLayout(grid_MAIN) - # ---- Progress Bar ---- + # ---- Progress Bar self.pbar = QProgressBar() self.pbar.setValue(0) self.pbar.hide() - # ---- Events ---- + # ---- Events # CORRELATION : - self.target_station.currentIndexChanged.connect(self.correlation_UI) self.distlimit.valueChanged.connect(self.correlation_table_display) self.altlimit.valueChanged.connect(self.correlation_table_display) self.date_start_widget.dateChanged.connect( @@ -439,14 +451,32 @@ def set_workdir(self, dirname): self.wxdata_merger.set_workdir(os.path.join(dirname, 'Meteo', 'Input')) - # ========================================================================= + def delete_current_dataset(self): + """ + Delete the current dataset source file and force a reload of the input + daily weather datafiles. + """ + current_index = self.target_station.currentIndex() + if current_index != -1: + basename = self.gapfill_worker.WEATHER.fnames[current_index] + dirname = self.gapfill_worker.inputDir + filename = os.path.join(dirname, basename) + delete_file(filename) + self.load_data_dir_content(reload=True) + + def btn_refresh_isclicked(self): + """ + Handles when the button to refresh the list of input daily weather + datafiles is clicked + """ + self.load_data_dir_content(reload=True) - def load_data_dir_content(self): - ''' + def load_data_dir_content(self, reload=False): + """ Initiate the loading of Weater Data Files contained in the - */Meteo/Input* folder and display the resulting station list in the - *Target station* combo box widget. - ''' + */Meteo/Input folder and display the resulting station list in the + target station combobox. + """ # Reset UI : @@ -460,10 +490,11 @@ def load_data_dir_content(self): self.CORRFLAG = 'off' # Correlation calculation won't be triggered when this is off - if self.sender() == self.btn_refresh_staList: + if reload: stanames = self.gapfill_worker.reload_data() else: stanames = self.gapfill_worker.load_data() + self.target_station.addItems(stanames) self.target_station.setCurrentIndex(-1) self.sta_display_summary.setHtml(self.gapfill_worker.read_summary()) @@ -535,16 +566,20 @@ def correlation_table_display(self): # =================================== self.FillTextBox.setText(table) self.target_station_info.setText(target_info) - def correlation_UI(self): # ============================================== + @QSlot(int) + def target_station_changed(self, index): + """Handles when the target station is changed on the GUI side.""" + self.btn_delete_data.setEnabled(index != -1) + if index != -1: + self.correlation_UI() + def correlation_UI(self): """ Calculate automatically the correlation coefficients when a target station is selected by the user in the drop-down menu or if a new station is selected programmatically. """ - if self.CORRFLAG == 'on' and self.target_station.currentIndex() != -1: - index = self.target_station.currentIndex() self.gapfill_worker.set_target_station(index) @@ -1529,8 +1564,7 @@ def sizeHint(self): app.setFont(QFont('Ubuntu', 11)) w = GapFillWeatherGUI() - w.set_workdir("C:\\Users\\jsgosselin\\OneDrive\\WHAT" - "\\WHAT\\tests\\@ new-prô'jèt!") + w.set_workdir("C:\\Users\\jsgosselin\\OneDrive\\GWHAT\\Projects\\Example") w.load_data_dir_content() lat = w.gapfill_worker.WEATHER.LAT diff --git a/gwhat/tests/test_gapfill_weather_data.py b/gwhat/tests/test_gapfill_weather_data.py new file mode 100644 index 000000000..11798d608 --- /dev/null +++ b/gwhat/tests/test_gapfill_weather_data.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Aug 4 01:50:50 2017 +@author: jsgosselin +""" + +# Standard library imports +import sys +import os + +# Third party imports +import numpy as np +from numpy import nan +import pytest +from PyQt5.QtCore import Qt + +# Local imports +sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +from gwhat.meteo.gapfill_weather_gui import GapFillWeatherGUI + + +# Qt Test Fixtures +# -------------------------------- + + +working_dir = os.path.join(os.getcwd(), "@ new-prô'jèt!") + + +@pytest.fixture +def gapfill_weather_bot(qtbot): + gapfiller = GapFillWeatherGUI() + gapfiller.set_workdir(working_dir) + qtbot.addWidget(gapfiller) + + return gapfiller, qtbot + + +# Test RawDataDownloader +# ------------------------------- + +expected_results = ["IBERVILLE", "IBERVILLE (1)", + "L'ACADIE", "L'ACADIE (1)", + "MARIEVILLE", "MARIEVILLE (1)", + "Station 1", "Station 1 (1)"] + + +@pytest.mark.run(order=5) +def test_refresh_data(gapfill_weather_bot, mocker): + gapfiller, qtbot = gapfill_weather_bot + gapfiller.show() + + # Load the input weather datafiles and assert that the list is loaded and + # displayed as expected. + qtbot.mouseClick(gapfiller.btn_refresh_staList, Qt.LeftButton) + + results = [] + for i in range(gapfiller.target_station.count()): + results.append(gapfiller.target_station.itemText(i)) + + assert expected_results == results + + +@pytest.mark.run(order=5) +def test_delete_data(gapfill_weather_bot, mocker): + gapfiller, qtbot = gapfill_weather_bot + gapfiller.show() + + # Load the input weather datafiles and select the last one in the list. + qtbot.mouseClick(gapfiller.btn_refresh_staList, Qt.LeftButton) + last_index = gapfiller.target_station.count()-1 + gapfiller.target_station.setCurrentIndex(last_index) + assert gapfiller.target_station.currentText() == expected_results[-1] + + # Delete the currently selected dataset. + qtbot.mouseClick(gapfiller.btn_delete_data, Qt.LeftButton) + + results = [] + for i in range(gapfiller.target_station.count()): + results.append(gapfiller.target_station.itemText(i)) + + assert expected_results[:-1] == results + + +if __name__ == "__main__": + pytest.main([os.path.basename(__file__)]) + # pytest.main() diff --git a/runtests.py b/runtests.py index 910631f4a..2eb69824c 100644 --- a/runtests.py +++ b/runtests.py @@ -9,6 +9,8 @@ """ import pytest +import matplotlib as mpl +mpl.use('Qt5Agg') def main():